[Jython-checkins] jython: PEP 3118 buffer API implemented for Jython.
jeff.allen
jython-checkins at python.org
Wed Jul 18 10:44:06 CEST 2012
http://hg.python.org/jython/rev/e0aa95bac30e
changeset: 6797:e0aa95bac30e
parent: 6787:cdf3485e1ba6
user: Jeff Allen <ja...py at farowl.co.uk>
date: Wed Jul 18 09:29:38 2012 +0100
summary:
PEP 3118 buffer API implemented for Jython.
This provides an implementation for one-dimensional buffers sufficient for use in the core. There is a thorough JUnit test. Types bytearray and str now have the API and memoryview is improved, although still not complete. Regression test scores: test_str (no errors), test_bytes (8 errors, unchanged), test_memoryview (37 errors, down a bit).
files:
src/org/python/core/BaseBytes.java | 105 +-
src/org/python/core/BufferPointer.java | 49 +
src/org/python/core/MemoryViewProtocol.java | 15 +-
src/org/python/core/Py.java | 5 +
src/org/python/core/PyBUF.java | 238 ++
src/org/python/core/MemoryView.java | 252 ++-
src/org/python/core/PyByteArray.java | 100 +-
src/org/python/core/PyMemoryView.java | 137 +-
src/org/python/core/PyString.java | 47 +-
src/org/python/core/buffer/BaseBuffer.java | 367 +++
src/org/python/core/buffer/SimpleBuffer.java | 66 +
src/org/python/core/buffer/SimpleReadonlyBuffer.java | 128 +
src/org/python/core/buffer/SimpleStringBuffer.java | 149 +
tests/java/org/python/core/BaseBytesTest.java | 70 +-
tests/java/org/python/core/PyBufferTest.java | 1094 ++++++++++
15 files changed, 2597 insertions(+), 225 deletions(-)
diff --git a/src/org/python/core/BaseBytes.java b/src/org/python/core/BaseBytes.java
--- a/src/org/python/core/BaseBytes.java
+++ b/src/org/python/core/BaseBytes.java
@@ -9,6 +9,8 @@
import java.util.List;
import java.util.ListIterator;
+import org.python.core.buffer.SimpleReadonlyBuffer;
+
/**
* Base class for Jython bytearray (and bytes in due course) that provides most of the Java API,
* including Java List behaviour. Attempts to modify the contents through this API will throw a
@@ -31,7 +33,7 @@
* </ul>
* since the default implementations will otherwise throw an exception.
*/
-public abstract class BaseBytes extends PySequence implements MemoryViewProtocol, List<PyInteger> {
+public abstract class BaseBytes extends PySequence implements List<PyInteger> {
/**
* Simple constructor of empty zero-length array of defined type.
@@ -141,75 +143,6 @@
/*
* ============================================================================================
- * Support for memoryview
- * ============================================================================================
- *
- * This is present in order to facilitate development of PyMemoryView which a full
- * implementation of bytearray would depend on, while at the same time a full implementation of
- * memoryview depends on bytearray.
- */
- /**
- * Get hold of a <code>memoryview</code> on the current byte array.
- *
- * @see MemoryViewProtocol#getMemoryView()
- */
- @Override
- public MemoryView getMemoryView() {
- if (mv == null) {
- mv = new MemoryViewImpl();
- }
- return mv;
- }
-
- private MemoryView mv;
-
- /**
- * All instances of BaseBytes have one dimension with stride one.
- */
- private static final PyTuple STRIDES = new PyTuple(Py.One);
-
- /**
- * Very simple MemoryView for one-dimensional byte array. This lacks any actual access to the
- * underlying storage as the interface is not presently defined.
- */
- private class MemoryViewImpl implements MemoryView {
-
- private final PyTuple SHAPE = new PyTuple(new PyInteger(storage.length));
-
- @Override
- public String get_format() {
- return "B";
- }
-
- @Override
- public int get_itemsize() {
- return 1;
- }
-
- @Override
- public PyTuple get_shape() {
- return SHAPE;
- }
-
- @Override
- public int get_ndim() {
- return 1;
- }
-
- @Override
- public PyTuple get_strides() {
- return STRIDES;
- }
-
- @Override
- public boolean get_readonly() {
- return true;
- }
-
- }
-
- /*
- * ============================================================================================
* Support for construction and initialisation
* ============================================================================================
*
@@ -257,11 +190,11 @@
*/
init((BaseBytes)arg);
- } else if (arg instanceof MemoryViewProtocol) {
+ } else if (arg instanceof BufferProtocol) {
/*
* bytearray copy of object supporting Jython implementation of PEP 3118
*/
- init(((MemoryViewProtocol)arg).getMemoryView());
+ init((BufferProtocol)arg);
} else {
/*
@@ -387,29 +320,21 @@
/**
* Helper for <code>__new__</code> and <code>__init__</code> and the Java API constructor from
- * objects supporting the Jython implementation of PEP 3118 (memoryview) in subclasses.
+ * objects supporting the Jython implementation of PEP 3118 (Buffer API) in subclasses.
*
- * @param value a memoryview object consistent with the slice assignment
- * @throws PyException(NotImplementedError) until memoryview is properly supported
- * @throws PyException(TypeError) if the memoryview is not byte-oriented
+ * @param value an object bearing the Buffer API and consistent with the slice assignment
*/
- protected void init(MemoryView value) throws PyException {
- // XXX Support memoryview once means of access to bytes is defined
- Py.NotImplementedError("memoryview not yet supported in bytearray");
- String format = value.get_format();
- boolean isBytes = format == null || "B".equals(format);
- if (value.get_ndim() != 1 || !isBytes) {
- Py.TypeError("memoryview value must be byte-oriented");
- } else {
- // Dimensions are given as a PyTuple (although only one)
- int len = value.get_shape().pyget(0).asInt();
- // XXX Access to memoryview bytes to go here
- }
+ protected void init(BufferProtocol value) throws PyException {
+ // Get the buffer view
+ PyBuffer view = value.getBuffer(PyBUF.SIMPLE);
+ // Create storage for the bytes and have the view drop them in
+ newStorage(view.getLen());
+ view.copyTo(storage, offset);
}
/**
- * Helper for the Java API constructor from a {@link #View}. View is (perhaps) a stop-gap while
- * there is no Jython implementation of PEP 3118 (memoryview).
+ * Helper for the Java API constructor from a {@link #View}. View is (perhaps) a stop-gap until
+ * the Jython implementation of PEP 3118 (buffer API) is embedded.
*
* @param value a byte-oriented view
*/
diff --git a/src/org/python/core/BufferPointer.java b/src/org/python/core/BufferPointer.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/BufferPointer.java
@@ -0,0 +1,49 @@
+package org.python.core;
+
+/**
+ * A class that references a contiguous slice of a <code>byte[]</code> array for use in the buffer
+ * API. This class simply bundles together a refernce to an array, a starting offset within that
+ * array, and specification of the number of bytes that may validly be accessed at that offset. It
+ * is used by the Jython buffer API roughly where the CPython buffer API uses a C (char *) pointer,
+ * or such a pointer and a length.
+ */
+public class BufferPointer {
+
+ /**
+ * Reference to the array holding the bytes. Usually this is the actual storage exported by a
+ * Python object. In some contexts the consumer will be entitled to make changes to the contents
+ * of this array, and in others, not. See {@link PyBuffer#isReadonly()}.
+ */
+ public final byte[] storage;
+ /** Starting position within the array for the data being pointed to. */
+ public final int offset;
+ /** Number of bytes within the array comprising the data being pointed to. */
+ public final int size;
+
+ /**
+ * Refer to a contiguous slice of the given array.
+ *
+ * @param storage array at reference
+ * @param offset index of the first byte
+ * @param size number of bytes being referred to
+ */
+ public BufferPointer(byte[] storage, int offset, int size) {
+ if ((offset | size | (storage.length-(offset + size))) < 0) {
+ throw Py.BufferError("Indexing error in buffer API");
+ }
+ this.storage = storage;
+ this.offset = offset;
+ this.size = size;
+ }
+
+ /**
+ * Refer to the whole of a byte array.
+ *
+ * @param storage array at reference
+ */
+ public BufferPointer(byte[] storage) {
+ this.storage = storage;
+ this.offset = 0;
+ this.size = storage.length;
+ }
+}
\ No newline at end of file
diff --git a/src/org/python/core/MemoryViewProtocol.java b/src/org/python/core/BufferProtocol.java
rename from src/org/python/core/MemoryViewProtocol.java
rename to src/org/python/core/BufferProtocol.java
--- a/src/org/python/core/MemoryViewProtocol.java
+++ b/src/org/python/core/BufferProtocol.java
@@ -1,6 +1,17 @@
package org.python.core;
-public interface MemoryViewProtocol {
+/**
+ * Interface marking an object as capable of exposing its internal state as a {@link PyBuffer}.
+ */
+public interface BufferProtocol {
- public MemoryView getMemoryView();
+ /**
+ * Method by which the consumer requests the buffer from the exporter. The consumer
+ * provides information on its intended method of navigation and the optional
+ * features the buffer object must provide.
+ *
+ * @param flags specification of options and the navigational capabilities of the consumer
+ * @return exported buffer
+ */
+ PyBuffer getBuffer(int flags);
}
diff --git a/src/org/python/core/Py.java b/src/org/python/core/Py.java
--- a/src/org/python/core/Py.java
+++ b/src/org/python/core/Py.java
@@ -363,7 +363,12 @@
public static PyException MemoryError(String message) {
return new PyException(Py.MemoryError, message);
}
+
public static PyObject BufferError;
+ public static PyException BufferError(String message) {
+ return new PyException(Py.BufferError, message);
+ }
+
public static PyObject ArithmeticError;
public static PyObject LookupError;
public static PyObject StandardError;
diff --git a/src/org/python/core/PyBUF.java b/src/org/python/core/PyBUF.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/PyBUF.java
@@ -0,0 +1,238 @@
+package org.python.core;
+
+/**
+ * This interface provides a base for the key interface of the buffer API, {@link PyBuffer},
+ * including symbolic constants used by the consumer of a <code>PyBuffer</code> to specify its
+ * requirements. The Jython buffer API emulates the CPython buffer API closely.
+ * <ul>
+ * <li>There are two reasons for separating parts of <code>PyBuffer</code> into this interface: The
+ * constants defined in CPython have the names <code>PyBUF_SIMPLE</code>,
+ * <code>PyBUF_WRITABLE</code>, etc., and the trick of defining ours here means we can write
+ * {@link PyBUF#SIMPLE}, {@link PyBUF#WRITABLE}, etc. so source code looks similar.</li>
+ * <li>It is not so easy in Java as it is in C to treat a <code>byte</code> array as storing
+ * anything other than <code>byte</code>, and we prepare for the possibility of buffers with a
+ * series of different primitive types by defining here, those methods that would be in common
+ * between <code>(Byte)Buffer</code> and an assumed future <code>FloatBuffer</code> or
+ * <code>TypedBuffer<T></code>. (Compare <code>java.nio.Buffer</code>.)</li>
+ * </ul>
+ * Except for other interfaces, it is unlikely any classes would implement <code>PyBUF</code>
+ * directly.
+ */
+public interface PyBUF {
+
+ /**
+ * Determine whether the consumer is entitled to write to the exported storage.
+ *
+ * @return true if writing is not allowed, false if it is.
+ */
+ boolean isReadonly();
+
+ /**
+ * The number of dimensions to the buffer. This number is the length of the <code>shape</code>
+ * array.
+ *
+ * @return number of dimensions
+ */
+ int getNdim();
+
+ /**
+ * An array reporting the size of the buffer, considered as a multidimensional array, in each
+ * dimension and (by its length) number of dimensions. The size is the size in "items". An item
+ * is the amount of buffer content addressed by one index or set of indices. In the simplest
+ * case an item is a single unit (byte), and there is one dimension. In complex cases, the array
+ * is multi-dimensional, and the item at each location is multi-unit (multi-byte). The consumer
+ * must not modify this array.
+ *
+ * @return the dimensions of the buffer as an array
+ */
+ int[] getShape();
+
+ /**
+ * The number of units (bytes) stored in each indexable item.
+ *
+ * @return the number of units (bytes) comprising each item.
+ */
+ int getItemsize();
+
+ /**
+ * The total number of units (bytes) stored, which will be the product of the elements of the
+ * shape, and the item size.
+ *
+ * @return the total number of units stored.
+ */
+ int getLen();
+
+ /**
+ * A buffer is (usually) coupled to the internal state of an exporting Python object, and that
+ * object may have to restrict its behaviour while the buffer exists. The consumer must
+ * therefore say when it has finished.
+ */
+ void release();
+
+ /**
+ * The "strides" array gives the distance in the storage array between adjacent items (in each
+ * dimension). If the rawest parts of the buffer API, the consumer of the buffer is able to
+ * navigate the exported storage. The "strides" array is part of the support for interpreting
+ * the buffer as an n-dimensional array of items. In the one-dimensional case, the "strides"
+ * array is In more dimensions, it provides the coefficients of the "addressing polynomial".
+ * (More on this in the CPython documentation.) The consumer must not modify this array.
+ *
+ * @return the distance in the storage array between adjacent items (in each dimension)
+ */
+ int[] getStrides();
+
+ /**
+ * The "suboffsets" array is a further part of the support for interpreting the buffer as an
+ * n-dimensional array of items, where the array potentially uses indirect addressing (like a
+ * real Java array of arrays, in fact). This is only applicable when there are more than 1
+ * dimension and works in conjunction with the <code>strides</code> array. (More on this in the
+ * CPython documentation.) When used, <code>suboffsets[k]</code> is an integer index, bit a byte
+ * offset as in CPython. The consumer must not modify this array.
+ *
+ * @return
+ */
+ int[] getSuboffsets();
+
+ /**
+ * Enquire whether the array is represented contiguously in the backing storage, according to C
+ * or Fortran ordering. A one-dimensional contiguous array is both.
+ *
+ * @param order 'C', 'F' or 'A', as the storage order is C, Fortran or either.
+ * @return true iff the array is stored contiguously in the order specified
+ */
+ boolean isContiguous(char order);
+
+ /* Constants taken from CPython object.h in v3.3.0a */
+
+ /**
+ * The maximum allowed number of dimensions (NumPy restriction?).
+ */
+ static final int MAX_NDIM = 64;
+ /**
+ * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
+ * specify that it expects to write to the buffer contents. getBuffer will raise an exception if
+ * the exporter's buffer cannot meet this requirement.
+ */
+ static final int WRITABLE = 0x0001;
+ /**
+ * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
+ * specify that it assumes a simple one-dimensional organisation of the exported storage with
+ * item size of one. getBuffer will raise an exception if the consumer sets this flag and the
+ * exporter's buffer cannot be navigated that simply.
+ */
+ static final int SIMPLE = 0;
+ /**
+ * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
+ * specify that it requires {@link PyBuffer#getFormat()} to return the type of the unit (rather
+ * than return <code>null</code>).
+ */
+ // I don't understand why we need this, or why format MUST be null of this is not set.
+ static final int FORMAT = 0x0004;
+ /**
+ * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
+ * specify that it it is prepared to navigate the buffer as multi-dimensional.
+ * <code>getBuffer</code> will raise an exception if consumer does not specify the flag but the
+ * exporter's buffer cannot be navigated without taking into account its multiple dimensions.
+ */
+ static final int ND = 0x0008 | SIMPLE; // Differs from CPython by or'ing in SIMPLE
+ /**
+ * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
+ * specify that it it expects to use the "strides" array. <code>getBuffer</code> will raise an
+ * exception if consumer does not specify the flag but the exporter's buffer cannot be navigated
+ * without using the "strides" array.
+ */
+ static final int STRIDES = 0x0010 | ND;
+ /**
+ * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
+ * specify that it will assume C-order organisation of the units. <code>getBuffer</code> will raise an
+ * exception if the exporter's buffer is not C-ordered. <code>C_CONTIGUOUS</code> implies
+ * <code>STRIDES</code>.
+ */
+ static final int C_CONTIGUOUS = 0x0020 | STRIDES;
+ /**
+ * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
+ * specify that it will assume Fortran-order organisation of the units. <code>getBuffer</code> will raise an
+ * exception if the exporter's buffer is not Fortran-ordered. <code>F_CONTIGUOUS</code> implies
+ * <code>STRIDES</code>.
+ */
+ static final int F_CONTIGUOUS = 0x0040 | STRIDES;
+ /**
+ * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
+ * specify that it
+ *
+ * getBuffer will raise an exception if the exporter's buffer is not contiguous.
+ * <code>ANY_CONTIGUOUS</code> implies <code>STRIDES</code>.
+ */
+ static final int ANY_CONTIGUOUS = 0x0080 | STRIDES;
+ /**
+ * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
+ * specify that it understands the "suboffsets" array. <code>getBuffer</code> will raise an
+ * exception if consumer does not specify the flag but the exporter's buffer cannot be navigated
+ * without understanding the "suboffsets" array. <code>INDIRECT</code> implies
+ * <code>STRIDES</code>.
+ */
+ static final int INDIRECT = 0x0100 | STRIDES;
+ /**
+ * Equivalent to <code>(ND | WRITABLE)</code>
+ */
+ static final int CONTIG = ND | WRITABLE;
+ /**
+ * Equivalent to <code>ND</code>
+ */
+ static final int CONTIG_RO = ND;
+ /**
+ * Equivalent to <code>(STRIDES | WRITABLE)</code>
+ */
+ static final int STRIDED = STRIDES | WRITABLE;
+ /**
+ * Equivalent to <code>STRIDES</code>
+ */
+ static final int STRIDED_RO = STRIDES;
+ /**
+ * Equivalent to <code>(STRIDES | WRITABLE | FORMAT)</code>
+ */
+ static final int RECORDS = STRIDES | WRITABLE | FORMAT;
+ /**
+ * Equivalent to <code>(STRIDES | FORMAT)</code>
+ */
+ static final int RECORDS_RO = STRIDES | FORMAT;
+ /**
+ * Equivalent to <code>(INDIRECT | WRITABLE | FORMAT)</code>
+ */
+ static final int FULL = INDIRECT | WRITABLE | FORMAT;
+ /**
+ * Equivalent to <code>(INDIRECT | FORMAT)</code>
+ */
+ static final int FULL_RO = INDIRECT | FORMAT;
+
+ /* Constants for readability, not standard for CPython */
+
+ /**
+ * Field mask, use as in <code>if ((capabilityFlags&ORGANISATION) == STRIDES) ...</code>.
+ */
+ static final int ORGANISATION = SIMPLE | ND | STRIDES | INDIRECT;
+ /**
+ * Field mask, use as in if <code>((capabilityFlags&ORGANIZATION) == STRIDES) ...</code>.
+ *
+ * @see #ORGANISATION
+ */
+ static final int ORGANIZATION = ORGANISATION;
+ /**
+ * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
+ * specify that it will assume C-order organisation of the units, irrespective of whether
+ * the strides array is to be provided. <code>getBuffer</code> will raise an
+ * exception if the exporter's buffer is not C-ordered. <code>C_CONTIGUOUS = IS_C_CONTIGUOUS | STRIDES</code>.
+ */
+ static final int IS_C_CONTIGUOUS = C_CONTIGUOUS & ~STRIDES;
+ /**
+ * A constant used by the consumer in its call to {@link BufferProtocol#getBuffer(int)} to
+ * specify that it will assume Fortran-order organisation of the units, irrespective of whether
+ * the strides array is to be provided. <code>getBuffer</code> will raise an
+ * exception if the exporter's buffer is not Fortran-ordered. <code>F_CONTIGUOUS = IS_F_CONTIGUOUS | STRIDES</code>.
+ */
+ static final int IS_F_CONTIGUOUS = F_CONTIGUOUS & ~STRIDES;
+ /**
+ * Field mask, use as in <code>if (capabilityFlags&CONTIGUITY== ... ) ...</code>.
+ */
+ static final int CONTIGUITY = (C_CONTIGUOUS | F_CONTIGUOUS | ANY_CONTIGUOUS) & ~STRIDES;
+}
\ No newline at end of file
diff --git a/src/org/python/core/MemoryView.java b/src/org/python/core/PyBuffer.java
rename from src/org/python/core/MemoryView.java
rename to src/org/python/core/PyBuffer.java
--- a/src/org/python/core/MemoryView.java
+++ b/src/org/python/core/PyBuffer.java
@@ -1,12 +1,248 @@
package org.python.core;
-public interface MemoryView {
- // readonly attributes XXX just the boring stuff so far
+/**
+ * The Jython buffer API for access to a byte array within an exporting object. This interface is
+ * the counterpart of the CPython <code>Py_buffer</code> struct. Several concrete types implement
+ * this interface in order to provide tailored support for different storage organisations.
+ */
+public interface PyBuffer extends PyBUF {
- public String get_format();
- public int get_itemsize();
- public PyTuple get_shape();
- public int get_ndim();
- public PyTuple get_strides();
- public boolean get_readonly();
+ /*
+ * The different behaviours required as the actual structure of the buffer changes (from one
+ * exporter to another, that is) should be dealt with using polymorphism. The implementation of
+ * those types may then calculate indices etc. without checking e.g for whether the strides
+ * array must be used, or the array is C or F contiguous, since they know the answer to these
+ * questions already, and can just get on with the request in their own way.
+ *
+ * The issue of consumer requests is different: the strides array will be present if the
+ * consumer asked for it, yet the methods of the buffer implementation do not have to use it
+ * (and won't).
+ */
+
+ // Informational methods inherited from PyBUF
+ //
+ // boolean isReadonly();
+ // int getNdim();
+ // int[] getShape();
+ // int getLen();
+
+ /**
+ * Return the byte indexed from a one-dimensional buffer with item size one. This is part of the
+ * fully-encapsulated API: the exporter takes care of navigating the structure of the buffer.
+ * Results are undefined where the number of dimensions is not one or if
+ * <code>itemsize>1</code>.
+ *
+ * @param index to retrieve from
+ * @return the item at index, which is a byte
+ */
+ byte byteAt(int index) throws IndexOutOfBoundsException;
+
+ /**
+ * Return the unsigned byte value indexed from a one-dimensional buffer with item size one. This
+ * is part of the fully-encapsulated API: the exporter takes care of navigating the structure of
+ * the buffer. Results are undefined where the number of dimensions is not one or if
+ * <code>itemsize>1</code>.
+ *
+ * @param index to retrieve from
+ * @return the item at index, treated as an unsigned byte, <code>=0xff & byteAt(index)</code>
+ */
+ int intAt(int index) throws IndexOutOfBoundsException;
+
+ /**
+ * Store the given byte at the indexed location in of a one-dimensional buffer with item size
+ * one. This is part of the fully-encapsulated API: the exporter takes care of navigating the
+ * structure of the buffer. Results are undefined where the number of dimensions is not one or
+ * if <code>itemsize>1</code>.
+ *
+ * @param value to store
+ * @param index to location
+ */
+ void storeAt(byte value, int index) throws IndexOutOfBoundsException;
+
+ // Access to n-dimensional array
+ //
+ /**
+ * Return the byte indexed from an N-dimensional buffer with item size one. This is part of the
+ * fully-encapsulated API: the exporter takes care of navigating the structure of the buffer.
+ * The indices must be correct in length and value for the array shape. Results are undefined
+ * where <code>itemsize>1</code>.
+ *
+ * @param indices specifying location to retrieve from
+ * @return the item at location, which is a byte
+ */
+ byte byteAt(int... indices) throws IndexOutOfBoundsException;
+
+ /**
+ * Return the unsigned byte value indexed from an N-dimensional buffer with item size one. This
+ * is part of the fully-encapsulated API: the exporter takes care of navigating the structure of
+ * the buffer. The indices must be correct in length and value for the array shape. Results are
+ * undefined where <code>itemsize>1</code>.
+ *
+ * @param index to retrieve from
+ * @return the item at location, treated as an unsigned byte, <code>=0xff & byteAt(index)</code>
+ */
+ int intAt(int... indices) throws IndexOutOfBoundsException;
+
+ /**
+ * Store the given byte at the indexed location in of an N-dimensional buffer with item size
+ * one. This is part of the fully-encapsulated API: the exporter takes care of navigating the
+ * structure of the buffer. The indices must be correct in length and value for the array shape.
+ * Results are undefined where <code>itemsize>1</code>.
+ *
+ * @param value to store
+ * @param indices specifying location to store at
+ */
+ void storeAt(byte value, int... indices) throws IndexOutOfBoundsException;
+
+ // Bulk access in one dimension
+ //
+ /**
+ * Copy the contents of the buffer to the destination byte array. The number of bytes will be
+ * that returned by {@link #getLen()}, and the order is the natural ordering according to the
+ * contiguity type.
+ *
+ * @param dest destination byte array
+ * @param destPos index in the destination array of the byte [0]
+ * @throws IndexOutOfBoundsException if the destination cannot hold it
+ */
+ void copyTo(byte[] dest, int destPos) throws IndexOutOfBoundsException;
+
+ /**
+ * Copy a simple slice of the buffer to the destination byte array, defined by a starting index
+ * and length in the source buffer. This may validly be done only for a one-dimensional buffer,
+ * as the meaning of the starting index is otherwise not defined.
+ *
+ * @param srcIndex starting index in the source buffer
+ * @param dest destination byte array
+ * @param destPos index in the destination array of the byte [0,...]
+ * @param length number of bytes to copy
+ * @throws IndexOutOfBoundsException if access out of bounds in source or destination
+ */
+ void copyTo(int srcIndex, byte[] dest, int destPos, int length) // mimic arraycopy args
+ throws IndexOutOfBoundsException;
+
+ /**
+ * Copy bytes from a slice of a (Java) byte array into the buffer. This may validly be done only
+ * for a one-dimensional buffer, as the meaning of the starting index is otherwise not defined.
+ *
+ * @param src source byte array
+ * @param srcPos location in source of first byte to copy
+ * @param destIndex starting index in the destination (i.e. <code>this</code>)
+ * @param length number of bytes to copy in
+ * @throws IndexOutOfBoundsException if access out of bounds in source or destination
+ * @throws PyException (BufferError) if read-only buffer
+ */
+ void copyFrom(byte[] src, int srcPos, int destIndex, int length) // mimic arraycopy args
+ throws IndexOutOfBoundsException, PyException;
+
+ // Bulk access in n-dimensions may be added here if desired semantics can be settled
+ //
+
+ // Buffer management inherited from PyBUF
+ //
+ // void release();
+
+ // Direct access to actual storage
+ //
+ /**
+ * Return a structure describing the slice of a byte array that holds the data being exported to
+ * the consumer. For a one-dimensional contiguous buffer, assuming the following client code
+ * where <code>obj</code> has type <code>BufferProtocol</code>:
+ *
+ * <pre>
+ *
+ * PyBuffer a = obj.getBuffer();
+ * int itemsize = a.getItemsize();
+ * BufferPointer b = a.getBuf();
+ * </pre>
+ *
+ * the item with index <code>k</code> is in the array <code>b.storage</code> at index
+ * <code>[b.offset + k*itemsize]</code> to <code>[b.offset + (k+1)*itemsize - 1]</code>
+ * inclusive. And if <code>itemsize==1</code>, the item is simply the byte
+ * <code>b.storage[b.offset + k]</code>
+ * <p>
+ * If the buffer is multidimensional or non-contiguous, <code>b.storage[b.offset]</code> is
+ * still the (first byte of) the item at index [0] or [0,...,0]. However, it is necessary to
+ * navigate <code>b</code> using the shape, strides and sub-offsets provided by the API.
+ *
+ * @return structure defining the byte[] slice that is the shared data
+ */
+ BufferPointer getBuf();
+
+ /**
+ * Return a structure describing the slice of a byte array that holds a single item from the
+ * data being exported to the consumer. For a one-dimensional contiguous buffer, assuming the
+ * following client code where <code>obj</code> has type <code>BufferProtocol</code>:
+ *
+ * <pre>
+ * int k = ... ;
+ * PyBuffer a = obj.getBuffer();
+ * int itemsize = a.getItemsize();
+ * BufferPointer b = a.getPointer(k);
+ * </pre>
+ *
+ * the item with index <code>k</code> is in the array <code>b.storage</code> at index
+ * <code>[b.offset]</code> to <code>[b.offset + itemsize - 1]</code> inclusive. And if
+ * <code>itemsize==1</code>, the item is simply the byte <code>b.storage[b.offset]</code>
+ * <p>
+ * Essentially this is a method for computing the offset of a particular index. Although
+ * <code>b.size==itemsize</code>, the client is free to navigate the underlying buffer
+ * <code>b.storage</code> without respecting these boundaries.
+ *
+ * @param index in the buffer to position the pointer
+ * @return structure defining the byte[] slice that is the shared data
+ */
+ BufferPointer getPointer(int index);
+
+ /**
+ * Return a structure describing the slice of a byte array that holds a single item from the
+ * data being exported to the consumer, in the case that array may be multi-dimensional. For an
+ * 3-dimensional contiguous buffer, assuming the following client code where <code>obj</code>
+ * has type <code>BufferProtocol</code>:
+ *
+ * <pre>
+ * int i, j, k = ... ;
+ * PyBuffer a = obj.getBuffer();
+ * int itemsize = a.getItemsize();
+ * BufferPointer b = a.getPointer(i,j,k);
+ * </pre>
+ *
+ * the item with index <code>[i,j,k]</code> is in the array <code>b.storage</code> at index
+ * <code>[b.offset]</code> to <code>[b.offset + itemsize - 1]</code> inclusive. And if
+ * <code>itemsize==1</code>, the item is simply the byte <code>b.storage[b.offset]</code>
+ * <p>
+ * Essentially this is a method for computing the offset of a particular index. Although
+ * <code>b.size==itemsize</code>, the client is free to navigate the underlying buffer
+ * <code>b.storage</code> without respecting these boundaries.
+ * <p>
+ * If the buffer is also non-contiguous, <code>b.storage[b.offset]</code> is still the (first
+ * byte of) the item at index [0,...,0]. However, it is necessary to navigate <code>b</code>
+ * using the shape, strides and sub-offsets provided by the API.
+ *
+ * @param indices multidimensional index at which to position the pointer
+ * @return structure defining the byte[] slice that is the shared data
+ */
+ BufferPointer getPointer(int... indices);
+
+ // Inherited from PyBUF and belonging here
+ //
+ // int[] getStrides();
+ // int[] getSuboffsets();
+ // boolean isContiguous(char order);
+
+ // Interpretation of bytes as items
+ /**
+ * A format string in the language of Python structs describing how the bytes of each item
+ * should be interpreted (or null if {@link PyBUF#FORMAT} was not part of the consumer's flags).
+ * <p>
+ * This is provided for compatibility with CPython. Jython only implements "B" so far, and it is
+ * debatable whether anything fancier than "<n>B" can be supported in Java.
+ *
+ * @return the format string
+ */
+ String getFormat();
+
+ // Inherited from PyBUF and belonging here
+ //
+ // int getItemsize();
}
diff --git a/src/org/python/core/PyByteArray.java b/src/org/python/core/PyByteArray.java
--- a/src/org/python/core/PyByteArray.java
+++ b/src/org/python/core/PyByteArray.java
@@ -2,6 +2,7 @@
import java.util.Arrays;
+import org.python.core.buffer.SimpleBuffer;
import org.python.expose.ExposedClassMethod;
import org.python.expose.ExposedMethod;
import org.python.expose.ExposedNew;
@@ -23,7 +24,7 @@
*
*/
@ExposedType(name = "bytearray", base = PyObject.class, doc = BuiltinDocs.bytearray_doc)
-public class PyByteArray extends BaseBytes {
+public class PyByteArray extends BaseBytes implements BufferProtocol {
public static final PyType TYPE = PyType.fromClass(PyByteArray.class);
@@ -85,14 +86,14 @@
}
/**
- * Create a new array filled exactly by a copy of the contents of the source, which is a
- * memoryview.
+ * Create a new array filled exactly by a copy of the contents of the source, which is an
+ * object supporting the Jython version of the PEP 3118 buffer API.
*
* @param value source of the bytes (and size)
*/
- public PyByteArray(MemoryViewProtocol value) {
+ public PyByteArray(BufferProtocol value) {
super(TYPE);
- init(value.getMemoryView());
+ init(value);
}
/**
@@ -196,6 +197,55 @@
init(arg);
}
+ /*
+ * ============================================================================================
+ * Support for the Buffer API
+ * ============================================================================================
+ *
+ * The buffer API allows other classes to access the storage directly.
+ */
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * The {@link PyBuffer} returned from this method is a one-dimensional array of single byte
+ * items, that allows modification of the object state but <b>prohibits resizing</b> the byte array.
+ * This prohibition is not only on the consumer of the view but extends to any other operations,
+ * such as any kind or insertion or deletion.
+ */
+ @Override
+ public synchronized PyBuffer getBuffer(int flags) {
+ exportCount++;
+ return new SimpleBuffer(this, new BufferPointer(storage, offset, size), flags) {
+
+ @Override
+ public void releaseAction() {
+ // synchronise on the same object as getBuffer()
+ synchronized (obj) {
+ exportCount--;
+ }
+ }
+ };
+ }
+
+ /**
+ * Test to see if the byte array may be resized and raise a BufferError if not.
+ *
+ * @throws PyException (BufferError) if there are buffer exports preventing a resize
+ */
+ protected void resizeCheck() throws PyException {
+ // XXX Quite likely this is not called in all the places it should be
+ if (exportCount!=0) {
+ throw Py.BufferError("Existing exports of data: object cannot be re-sized");
+ }
+ }
+
+ /**
+ * Count of PyBuffer exports not yet released, used to prevent untimely resizing.
+ */
+ private int exportCount;
+
+
/* ============================================================================================
* API for org.python.core.PySequence
* ============================================================================================
@@ -368,12 +418,12 @@
*/
setslice(start, stop, step, (BaseBytes)value);
- } else if (value instanceof MemoryViewProtocol) {
+ } else if (value instanceof BufferProtocol) {
/*
* Value supports Jython implementation of PEP 3118, and can be can be inserted without
* making a copy.
*/
- setslice(start, stop, step, ((MemoryViewProtocol)value).getMemoryView());
+ setslice(start, stop, step, (BufferProtocol)value);
} else {
/*
@@ -449,12 +499,31 @@
* @param start the position of the first element.
* @param stop one more than the position of the last element.
* @param step the step size.
- * @param value a memoryview object consistent with the slice assignment
+ * @param value an object supporting the buffer API consistent with the slice assignment
* @throws PyException(SliceSizeError) if the value size is inconsistent with an extended slice
*/
- private void setslice(int start, int stop, int step, MemoryView value) throws PyException {
- // XXX Support memoryview once means of access to bytes is defined
- throw Py.NotImplementedError("memoryview not yet supported in bytearray");
+ private void setslice(int start, int stop, int step, BufferProtocol value) throws PyException {
+ PyBuffer view = value.getBuffer(PyBUF.SIMPLE);
+
+
+ int len = view.getLen();
+
+ if (step == 1) {
+ // Delete this[start:stop] and open a space of the right size
+ storageReplace(start, stop - start, len);
+ view.copyTo(storage, start+offset);
+
+ } else {
+ // This is an extended slice which means we are replacing elements
+ int n = sliceLength(start, stop, step);
+ if (n != len) {
+ throw SliceSizeError("bytes", len, n);
+ }
+
+ for (int io = start + offset, j = 0; j < n; io += step, j++) {
+ storage[io] = view.byteAt(j); // Assign this[i] = value[j]
+ }
+ }
}
/**
@@ -2089,6 +2158,9 @@
return;
}
+ // This will not be possible if this object has buffer exports
+ resizeCheck();
+
// Compute some handy points of reference
final int L = storage.length;
final int f = offset;
@@ -2351,6 +2423,9 @@
return; // Everything stays where it is.
}
+ // This will not be possible if this object has buffer exports
+ resizeCheck();
+
// Compute some handy points of reference
final int L = storage.length;
final int f = offset;
@@ -2430,6 +2505,9 @@
return; // Everything stays where it is.
}
+ // This will not be possible if this object has buffer exports
+ resizeCheck();
+
// Compute some handy points of reference
final int L = storage.length;
final int f = offset;
diff --git a/src/org/python/core/PyMemoryView.java b/src/org/python/core/PyMemoryView.java
--- a/src/org/python/core/PyMemoryView.java
+++ b/src/org/python/core/PyMemoryView.java
@@ -1,61 +1,140 @@
package org.python.core;
import org.python.expose.ExposedGet;
-import org.python.expose.ExposedMethod;
import org.python.expose.ExposedNew;
import org.python.expose.ExposedType;
+/**
+ * Class implementing the Python <code>memoryview</code> type, at present highly incomplete. It
+ * provides a wrapper around the Jython buffer API, but slice operations and most others are
+ * missing.
+ */
@ExposedType(name = "memoryview", base = PyObject.class, isBaseType = false)
public class PyMemoryView extends PyObject {
+ // XXX This should probably extend PySequence to get the slice behaviour
+
public static final PyType TYPE = PyType.fromClass(PyMemoryView.class);
- MemoryView backing;
+ /**
+ * The buffer exported by the object. We do not a present implement the buffer sharing strategy
+ * used by CPython <code>memoryview</code>.
+ */
+ private PyBuffer backing;
+ /** Cache the result of getting shape here. */
+ private PyTuple shape;
+ /** Cache the result of getting strides here. */
+ private PyTuple strides;
- public PyMemoryView(MemoryViewProtocol obj) {
- backing = obj.getMemoryView();
+ /**
+ * Construct a PyMemoryView from an object that bears the necessary BufferProtocol interface.
+ * The buffer so obtained will be writable if the underlying object permits it.
+ *
+ * @param obj object that will export the buffer
+ */
+ public PyMemoryView(BufferProtocol obj) {
+ /*
+ * Ask for the full set of facilities (strides, indirect, etc.) from the object in case they
+ * are necessary for navigation, but only ask for read access. If the object is writable,
+ * the PyBuffer will be writable.
+ */
+ backing = obj.getBuffer(PyBUF.FULL_RO);
}
@ExposedNew
- static PyObject memoryview_new(PyNewWrapper new_, boolean init, PyType subtype, PyObject[] args,
- String[] keywords) {
+ static PyObject memoryview_new(PyNewWrapper new_, boolean init, PyType subtype,
+ PyObject[] args, String[] keywords) {
PyObject obj = args[0];
- if (obj instanceof MemoryViewProtocol) {
- return new PyMemoryView((MemoryViewProtocol)obj);
+ if (obj instanceof BufferProtocol) {
+ return new PyMemoryView((BufferProtocol)obj);
+ } else {
+ throw Py.TypeError("cannot make memory view because object does not have "
+ + "the buffer interface");
}
- else throw Py.TypeError("cannot make memory view because object does not have the buffer interface");
}
- @ExposedGet(name = "format")
- public String get_format() {
- return backing.get_format();
+ @ExposedGet(doc = format_doc)
+ public String format() {
+ return backing.getFormat();
}
- @ExposedGet(name = "itemsize")
- public int get_itemsize() {
- return backing.get_itemsize();
+ @ExposedGet(doc = itemsize_doc)
+ public int itemsize() {
+ return backing.getItemsize();
}
- @ExposedGet(name = "shape")
- public PyTuple get_shape() {
- return backing.get_shape();
+ @ExposedGet(doc = shape_doc)
+ public PyTuple shape() {
+ if (shape == null) {
+ shape = tupleOf(backing.getShape());
+ }
+ return shape;
}
- @ExposedGet(name = "ndim")
- public int get_ndim() {
- return backing.get_ndim();
+ @ExposedGet(doc = ndim_doc)
+ public int ndim() {
+ return backing.getShape().length;
}
- @ExposedGet(name = "strides")
- public PyTuple get_strides() {
- return backing.get_strides();
+ @ExposedGet(doc = strides_doc)
+ public PyTuple strides() {
+ if (strides == null) {
+ strides = tupleOf(backing.getStrides());
+ }
+ return strides;
}
- @ExposedGet(name = "readonly")
- public boolean get_readonly() {
- return backing.get_readonly();
+ @ExposedGet(doc = readonly_doc)
+ public boolean readonly() {
+ return backing.isReadonly();
}
+ /**
+ * Make an integer array into a PyTuple of PyInteger values.
+ *
+ * @param x the array
+ * @return the PyTuple
+ */
+ private PyTuple tupleOf(int[] x) {
+ PyInteger[] pyx = new PyInteger[x.length];
+ for (int k = 0; k < x.length; k++) {
+ pyx[k] = new PyInteger(x[k]);
+ }
+ return new PyTuple(pyx, false);
+ }
+
+ /*
+ * These strings are adapted from the on-line documentation as the attributes do not come with
+ * any docstrings.
+ */
+ private final static String memoryview_tobytes_doc = "tobytes()\n\n"
+ + "Return the data in the buffer as a bytestring (an object of class str).\n\n"
+ + ">>> m = memoryview(\"abc\")\n" + ">>> m.tobytes()\n" + "'abc'";
+
+ private final static String memoryview_tolist_doc = "tolist()\n\n"
+ + "Return the data in the buffer as a list of integers.\n\n"
+ + ">>> memoryview(\"abc\").tolist()\n" + "[97, 98, 99]";
+
+ private final static String format_doc = "format\n"
+ + "A string containing the format (in struct module style) for each element in\n"
+ + "the view. This defaults to 'B', a simple bytestring.\n";
+
+ private final static String itemsize_doc = "itemsize\n"
+ + "The size in bytes of each element of the memoryview.\n";
+
+ private final static String shape_doc = "shape\n"
+ + "A tuple of integers the length of ndim giving the shape of the memory as an\n"
+ + "N-dimensional array.\n";
+
+ private final static String ndim_doc = "ndim\n"
+ + "An integer indicating how many dimensions of a multi-dimensional array the\n"
+ + "memory represents.\n";
+
+ private final static String strides_doc = "strides\n"
+ + "A tuple of integers the length of ndim giving the size in bytes to access\n"
+ + "each element for each dimension of the array.\n";
+
+ private final static String readonly_doc = "readonly\n"
+ + "A bool indicating whether the memory is read only.\n";
+
}
-
-
diff --git a/src/org/python/core/PyString.java b/src/org/python/core/PyString.java
--- a/src/org/python/core/PyString.java
+++ b/src/org/python/core/PyString.java
@@ -1,6 +1,12 @@
/// Copyright (c) Corporation for National Research Initiatives
package org.python.core;
+import java.math.BigInteger;
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
+
+import org.python.core.StringFormatter.DecimalFormatTemplate;
+import org.python.core.buffer.SimpleStringBuffer;
import org.python.core.stringlib.FieldNameIterator;
import org.python.core.stringlib.InternalFormatSpec;
import org.python.core.stringlib.InternalFormatSpecParser;
@@ -12,15 +18,11 @@
import org.python.expose.ExposedType;
import org.python.expose.MethodType;
-import java.math.BigInteger;
-import java.text.DecimalFormat;
-import java.text.DecimalFormatSymbols;
-
/**
* A builtin python string.
*/
@ExposedType(name = "str", doc = BuiltinDocs.str_doc)
-public class PyString extends PyBaseString implements MemoryViewProtocol
+public class PyString extends PyBaseString implements BufferProtocol
{
public static final PyType TYPE = PyType.fromClass(PyString.class);
protected String string; // cannot make final because of Python intern support
@@ -93,28 +95,19 @@
return codePoints;
}
- public MemoryView getMemoryView() {
- return new MemoryView() {
- // beginning of support
- public String get_format() {
- return "B";
- }
- public int get_itemsize() {
- return 2;
- }
- public PyTuple get_shape() {
- return new PyTuple(Py.newInteger(getString().length()));
- }
- public int get_ndim() {
- return 1;
- }
- public PyTuple get_strides() {
- return new PyTuple(Py.newInteger(1));
- }
- public boolean get_readonly() {
- return true;
- }
- };
+ /**
+ * Create a read-only buffer view of the contents of the string, treating it as a sequence of
+ * unsigned bytes. The caller specifies its requirements and navigational capabilities in the
+ * <code>flags</code> argument (see the constants in class {@link PyBUF} for an explanation).
+ *
+ * @param flags consumer requirements
+ * @return the requested buffer
+ */
+ public PyBuffer getBuffer(int flags) {
+ /*
+ * Return a buffer, but specialised to defer construction of the buf object.
+ */
+ return new SimpleStringBuffer(this, getString(), flags);
}
public String substring(int start, int end) {
diff --git a/src/org/python/core/buffer/BaseBuffer.java b/src/org/python/core/buffer/BaseBuffer.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/buffer/BaseBuffer.java
@@ -0,0 +1,367 @@
+package org.python.core.buffer;
+
+import org.python.core.BufferPointer;
+import org.python.core.BufferProtocol;
+import org.python.core.Py;
+import org.python.core.PyBUF;
+import org.python.core.PyBuffer;
+import org.python.core.PyException;
+
+/**
+ * Base implementation of the Buffer API for implementations to extend. The default implementation
+ * provides some mechanisms for checking the consumer's capabilities against those stated as
+ * necessary by the exporter. Default implementations of methods are provided for the standard array
+ * organisations. The implementors of simple buffers will find it more efficient to override methods
+ * to which performance might be sensitive with a calculation specific to their actual type.
+ * <p>
+ * The default implementation raises a read-only exception for those methods that store data in the
+ * buffer, and {@link #isReadonly()} returns <code>true</code>. Writable types must override this
+ * implementation. Default implementations of other methods are generally oriented towards
+ * contiguous N-dimensional arrays.
+ * <p>
+ * At the time of writing, only the SIMPLE organisation (one-dimensional, of item size one) is used
+ * in the Jython core.
+ */
+public abstract class BaseBuffer implements PyBuffer {
+
+ /**
+ * The object from which this buffer export must be released (see {@link PyBuffer#release()}).
+ * This is normally the original exporter of this buffer and the owner of the underlying
+ * storage. Exceptions to this occur when some other object is managing release (this is the
+ * case when a <code>memoryview</code> has provided the buffer), and when disposal can safely be
+ * left to the Java garbage collector (local temporaries and perhaps exports from immutable
+ * objects).
+ */
+ protected BufferProtocol obj;
+ /**
+ * The dimensions of the array represented by the buffer. The length of the <code>shape</code>
+ * array is the number of dimensions. The <code>shape</code> array should always be created and
+ * filled (difference from CPython).
+ */
+ protected int[] shape;
+ /**
+ * Step sizes in the underlying buffer essential to correct translation of an index (or indices)
+ * into an index into the storage. This reference will be <code>null</code> if not needed for
+ * the storage organisation, and not requested by the consumer in <code>flags</code>. If it is
+ * either necessary for the buffer navigation, or requested by the consumer in flags, the
+ * <code>strides</code> array must be correctly filled to at least the length of the
+ * <code>shape</code> array.
+ */
+ protected int[] strides;
+ /**
+ * Reference to a structure that wraps the underlying storage that the exporter is sharing with
+ * the consumer.
+ */
+ protected BufferPointer buf;
+ /**
+ * Bit pattern using the constants defined in {@link PyBUF} that records the actual capabilities
+ * this buffer offers. See {@link #assignCapabilityFlags(int, int, int, int)}.
+ */
+ protected int capabilityFlags;
+
+ /**
+ * The result of the operation is to set the {@link #capabilityFlags} according to the
+ * capabilities this instance should support. This method is normally called in the constructor
+ * of each particular sub-class of <code>BaseBuffer</code>, passing in a <code>flags</code>
+ * argument that originated in the consumer's call to {@link BufferProtocol#getBuffer(int)}.
+ * <p>
+ * The consumer supplies as a set of <code>flags</code>, using constants from {@link PyBUF}, the
+ * capabilities that it expects from the buffer. These include a statement of which navigational
+ * arrays it will use ( <code>shape</code>, <code>strides</code>, and <code>suboffsets</code>),
+ * whether it wants the <code>format</code> string set so it describes the item type or left
+ * null, and whether it expects the buffer to be writable. The consumer flags are taken by this
+ * method both as a statement of needs to be met by the buffer, and as a statement of
+ * capabilities in the consumer to navigate different buffers.
+ * <p>
+ * In its call to this method, the exporter specifies the capabilities it requires the consumer
+ * to have (and indicate by asking for them in <code>flags</code>) in order to navigate the
+ * buffer successfully. For example, if the buffer is a strided array, the consumer must specify
+ * that it expects the <code>strides</code> array. Otherwise the method concludes the consumer
+ * is not capable of the navigation required. Capabilities specified in the
+ * <code>requiredFlags</code> must appear in the consumer's <code>flags</code> request. If any
+ * don't, a Python <code>BufferError</code> will be raised. If there is no error these flags
+ * will be set in <code>capabilityFlags</code> as required of the buffer.
+ * <p>
+ * The exporter specifies some capabilities it <i>allows</i> the consumer to request, such as
+ * the <code>format</code> string. Depending on the type of exporter, the navigational arrays (
+ * <code>shape</code>, <code>strides</code>, and <code>suboffsets</code>) may also be allowed
+ * rather than required. Capabilities specified in the <code>allowedFlags</code>, if they also
+ * appear in the consumer's <code>flags</code>, will be set in <code>capabilityFlags</code>.
+ * <p>
+ * The exporter specifies some capabilities that will be supplied whether requested or not. For
+ * example (and it might be the only one) this is used only to express that an unstrided,
+ * one-dimensional array is <code>C_CONTIGUOUS</code>, <code>F_CONTIGUOUS</code>, and
+ * <code>ANY_CONTIGUOUS</code>, all at once. Capabilities specified in the
+ * <code>impliedFlags</code>, will be set in <code>capabilityFlags</code> whether in the
+ * consumer's <code>flags</code> or not.
+ * <p>
+ * Capabilities specified in the consumer's <code>flags</code> request, if they do not appear in
+ * the exporter's <code>requiredFlags</code> <code>allowedFlags</code> or
+ * <code>impliedFlags</code>, will cause a Python <code>BufferError</code>.
+ * <p>
+ * Note that this method cannot actually set the <code>shape</code>, <code>strides</code> and
+ * <code>suboffsets</code> properties: the implementation of the specific buffer type must do
+ * that based on the <code>capabilityFlags</code>. This forms a partial counterpart to CPython
+ * <code>PyBuffer_FillInfo()</code> but it is not specific to the simple type of buffer, and
+ * covers the flag processing of all buffer types. This is complex (in CPython) and the Jython
+ * approach attempts to be compatible yet comprehensible.
+ */
+ protected void assignCapabilityFlags(int flags, int requiredFlags, int allowedFlags,
+ int impliedFlags) {
+
+ // Ensure what may be requested includes what must be and what comes unasked
+ allowedFlags = allowedFlags | requiredFlags | impliedFlags;
+
+ // Look for request flags (other than buffer organisation) outside what is allowed
+ int syndrome = flags & ~(allowedFlags | ORGANISATION);
+
+ if (syndrome != 0) {
+ // Some flag was set that is neither required nor allowed
+ if ((syndrome & WRITABLE) != 0) {
+ throw notWritable();
+ } else if ((syndrome & C_CONTIGUOUS) != 0) {
+ throw bufferIsNot("C-contiguous");
+ } else if ((syndrome & F_CONTIGUOUS) != 0) {
+ throw bufferIsNot("Fortran-contiguous");
+ } else if ((syndrome & ANY_CONTIGUOUS) != 0) {
+ throw bufferIsNot("contiguous");
+ } else {
+ // Catch-all error (never in practice?)
+ throw bufferIsNot("capable of matching request");
+ }
+
+ } else if ((flags & requiredFlags) != requiredFlags) {
+ // This buffer needs more capability to navigate than the consumer has requested
+ if ((flags & ND) != ND) {
+ throw bufferRequires("shape");
+ } else if ((flags & STRIDES) != STRIDES) {
+ throw bufferRequires("strides");
+ } else if ((flags & INDIRECT) != INDIRECT) {
+ throw bufferRequires("suboffsets");
+ } else {
+ // Catch-all error
+ throw bufferRequires("feature consumer lacks");
+ }
+
+ } else {
+ // These flags control returns from (default) getShape etc..
+ capabilityFlags = (flags & allowedFlags) | impliedFlags;
+ // Note that shape and strides are still to be initialised
+ }
+
+ /*
+ * Caller must responds to the requested/required capabilities with shape and strides arrays
+ * suited to the actual type of buffer.
+ */
+ }
+
+ /**
+ * Provide an instance of BaseBuffer or a sub-class meeting the consumer's expectations as
+ * expressed in the flags argument. Compare CPython:
+ *
+ * <pre>
+ * int PyBuffer_FillInfo(Py_buffer *view, PyObject *exporter,
+ * void *buf, Py_ssize_t len,
+ * int readonly, int flags)
+ * </pre>
+ *
+ * @param exporter the exporting object
+ * @param buf descriptor for the exported buffer itself
+ */
+ protected BaseBuffer(BufferProtocol exporter, BufferPointer buf) {
+ // Exporting object (is allowed to be null)
+ this.obj = exporter;
+ // Exported data (not normally allowed to be null)
+ this.buf = buf;
+ }
+
+ @Override
+ public boolean isReadonly() {
+ // Default position is read only: mutable buffers must override
+ return true;
+ }
+
+ @Override
+ public int getNdim() {
+ return shape.length;
+ }
+
+ @Override
+ public int[] getShape() {
+ // Difference from CPython: never null, even when the consumer doesn't request it.
+ return shape;
+ }
+
+ @Override
+ public int getLen() {
+ // Correct if contiguous. Override if strided or indirect with itemsize*product(shape).
+ return buf.size;
+ }
+
+ // Let the sub-class implement:
+ // @Override public byte byteAt(int index) throws IndexOutOfBoundsException {}
+
+ @Override
+ public int intAt(int index) throws IndexOutOfBoundsException {
+ return 0xff & byteAt(index);
+ }
+
+ @Override
+ public void storeAt(byte value, int index) throws IndexOutOfBoundsException, PyException {
+ throw notWritable();
+ }
+
+ // Let the sub-class implement:
+ // @Override public byte byteAt(int... indices) throws IndexOutOfBoundsException {}
+
+ @Override
+ public int intAt(int... indices) throws IndexOutOfBoundsException {
+ return 0xff & byteAt(indices);
+ }
+
+ @Override
+ public void storeAt(byte value, int... indices) throws IndexOutOfBoundsException, PyException {
+ throw notWritable();
+ }
+
+ @Override
+ public void copyTo(byte[] dest, int destPos) throws IndexOutOfBoundsException {
+ // Correct for contiguous arrays (if destination expects same F or C contiguity)
+ copyTo(0, dest, destPos, getLen());
+ }
+
+ // Let the sub-class implement:
+ // @Override public void copyTo(int srcIndex, byte[] dest, int destPos, int length)
+ // throws IndexOutOfBoundsException {}
+
+ @Override
+ public void copyFrom(byte[] src, int srcPos, int destIndex, int length)
+ throws IndexOutOfBoundsException, PyException {
+ throw notWritable();
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * The implementation here calls {@link #releaseAction()}, which the implementer of a specific
+ * buffer type should override with the necessary actions to release the buffer from the
+ * exporter. It is not an error to call this method more than once (difference from CPython), or
+ * on a temporary buffer that needs no release action. If not released explicitly, it will be
+ * called during object finalisation (before garbage collection) of the buffer object.
+ */
+ @Override
+ public final void release() {
+ if (obj != null) {
+ releaseAction();
+ }
+ obj = null;
+ }
+
+ @Override
+ public BufferPointer getBuf() {
+ return buf;
+ }
+
+ // Let the sub-class implement:
+ // @Override public BufferPointer getPointer(int index) { return null; }
+ // @Override public BufferPointer getPointer(int... indices) { return null; }
+
+ @Override
+ public int[] getStrides() {
+ return strides;
+ }
+
+ @Override
+ public int[] getSuboffsets() {
+ return null;
+ }
+
+ @Override
+ public boolean isContiguous(char order) {
+ return true;
+ }
+
+ @Override
+ public String getFormat() {
+ // Avoid having to have an actual 'format' member
+ return ((capabilityFlags & FORMAT) == 0) ? null : "B";
+ }
+
+ @Override
+ public int getItemsize() {
+ // Avoid having to have an actual 'itemsize' member
+ return 1;
+ }
+
+ /**
+ * Ensure buffer, if not released sooner, is released from the exporter during object
+ * finalisation (before garbage collection) of the buffer object.
+ */
+ @Override
+ protected void finalize() throws Throwable {
+ release();
+ super.finalize();
+ }
+
+ /**
+ * This method will be called when the consumer calls {@link #release()} (to be precise, only on
+ * the first call). The default implementation does nothing. Override this method to add release
+ * behaviour specific to exporter. A common convention is to do this within the definition of
+ * {@link BufferProtocol#getBuffer(int)} within the exporting class, where a nested class is
+ * finally defined.
+ */
+ protected void releaseAction() {}
+
+ /**
+ * Check the number of indices (but not their values), raising a Python BufferError if this does
+ * not match the number of dimensions.
+ *
+ * @param indices into the buffer (to test)
+ * @return number of dimensions
+ * @throws PyException (BufferError) if wrong number of indices
+ */
+ final int checkDimension(int[] indices) throws PyException {
+ int ndim = shape.length;
+ if (indices.length != ndim) {
+ if (indices.length < ndim) {
+ throw Py.BufferError("too few indices supplied");
+ } else {
+ throw Py.BufferError("too many indices supplied");
+ }
+ }
+ return ndim;
+ }
+
+ /**
+ * Convenience method to create (for the caller to throw) a
+ * <code>BufferError("underlying buffer is not writable")</code>.
+ *
+ * @return the error as a PyException
+ */
+ protected PyException notWritable() {
+ return bufferIsNot("writable");
+ }
+
+ /**
+ * Convenience method to create (for the caller to throw) a
+ * <code>BufferError("underlying buffer is not {property}")</code>.
+ *
+ * @param property
+ * @return the error as a PyException
+ */
+ protected PyException bufferIsNot(String property) {
+ return Py.BufferError("underlying buffer is not " + property);
+ }
+
+ /**
+ * Convenience method to create (for the caller to throw) a
+ * <code>BufferError("underlying buffer requires {feature}")</code>.
+ *
+ * @param feature
+ * @return the error as a PyException
+ */
+ protected PyException bufferRequires(String feature) {
+ return Py.BufferError("underlying buffer requires " + feature);
+ }
+
+}
diff --git a/src/org/python/core/buffer/SimpleBuffer.java b/src/org/python/core/buffer/SimpleBuffer.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/buffer/SimpleBuffer.java
@@ -0,0 +1,66 @@
+package org.python.core.buffer;
+
+import org.python.core.BufferPointer;
+import org.python.core.BufferProtocol;
+
+/**
+ * Buffer API over a writable one-dimensional array of one-byte items.
+ */
+public class SimpleBuffer extends SimpleReadonlyBuffer {
+
+ /**
+ * <code>SimpleBuffer</code> allows consumer requests that are the same as
+ * <code>SimpleReadonlyBuffer</code>, with the addition of WRITABLE.
+ */
+ protected static final int ALLOWED_FLAGS = WRITABLE | SimpleReadonlyBuffer.ALLOWED_FLAGS;
+
+ /**
+ * Provide an instance of <code>SimpleBuffer</code> in a default, semi-constructed state. The
+ * sub-class constructor takes responsibility for completing construction with a call to
+ * {@link #assignCapabilityFlags(int, int, int, int)}.
+ *
+ * @param exporter the exporting object
+ * @param buf wrapping the array of bytes storing the implementation of the object
+ */
+ protected SimpleBuffer(BufferProtocol exporter, BufferPointer buf) {
+ super(exporter, buf);
+ }
+
+ /**
+ * Provide an instance of SimpleBuffer meeting the consumer's expectations as expressed in the
+ * flags argument.
+ *
+ * @param exporter the exporting object
+ * @param buf wrapping the array of bytes storing the implementation of the object
+ * @param flags consumer requirements
+ */
+ public SimpleBuffer(BufferProtocol exporter, BufferPointer buf, int flags) {
+ super(exporter, buf);
+ assignCapabilityFlags(flags, REQUIRED_FLAGS, ALLOWED_FLAGS, IMPLIED_FLAGS);
+ fillInfo();
+ }
+
+ @Override
+ public boolean isReadonly() {
+ return false;
+ }
+
+ @Override
+ public void storeAt(byte value, int index) {
+ buf.storage[buf.offset + index] = value;
+ }
+
+ @Override
+ public void storeAt(byte value, int... indices) {
+ if (indices.length != 1) {
+ checkDimension(indices);
+ }
+ storeAt(value, indices[0]);
+ }
+
+ @Override
+ public void copyFrom(byte[] src, int srcPos, int destIndex, int length) {
+ System.arraycopy(src, srcPos, buf.storage, buf.offset + destIndex, length);
+ }
+
+}
diff --git a/src/org/python/core/buffer/SimpleReadonlyBuffer.java b/src/org/python/core/buffer/SimpleReadonlyBuffer.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/buffer/SimpleReadonlyBuffer.java
@@ -0,0 +1,128 @@
+package org.python.core.buffer;
+
+import org.python.core.BufferPointer;
+import org.python.core.BufferProtocol;
+
+/**
+ * Buffer API over a one-dimensional array of one-byte items providing read-only API. A writable
+ * simple buffer will extend this implementation.
+ */
+public class SimpleReadonlyBuffer extends BaseBuffer {
+
+ /**
+ * Using the PyBUF constants, express capabilities the consumer must request if it is to
+ * navigate the storage successfully. (None.)
+ */
+ public static final int REQUIRED_FLAGS = 0;
+ /**
+ * Using the PyBUF constants, express capabilities the consumer may request so it can navigate
+ * the storage in its chosen way. The buffer instance has to implement these mechanisms if and
+ * only if they are requested. (FORMAT | ND | STRIDES | INDIRECT)
+ */
+ public static final int ALLOWED_FLAGS = FORMAT | ND | STRIDES | INDIRECT;
+ /**
+ * Using the PyBUF constants, express capabilities the consumer doesn't need to request because
+ * they will be there anyway. (One-dimensional arrays (including those sliced with step size
+ * one) are C- and F-contiguous.)
+ */
+ public static final int IMPLIED_FLAGS = CONTIGUITY;
+ /**
+ * The strides array for this type is always a single element array with a 1 in it.
+ */
+ protected static final int[] SIMPLE_STRIDES = {1};
+
+ /**
+ * Partial counterpart to CPython <code>PyBuffer_FillInfo()</code> specific to the simple type
+ * of buffer and called from the constructor. The base constructor will already have been
+ * called, filling {@link #buf} and {@link #obj}. And the method
+ * {@link #assignCapabilityFlags(int, int, int, int)} has set {@link #capabilityFlags}.
+ */
+ protected void fillInfo() {
+ /*
+ * We will already have called: assignCapabilityFlags(flags, requiredFlags, allowedFlags,
+ * impliedFlags); So capabilityFlags holds the requests for shape, strides, writable, etc..
+ */
+ // Difference from CPython: never null, even when the consumer doesn't request it
+ shape = new int[1];
+ shape[0] = getLen();
+
+ // Following CPython: provide strides only when the consumer requests it
+ if ((capabilityFlags & STRIDES) == STRIDES) {
+ strides = SIMPLE_STRIDES;
+ }
+
+ // Even when the consumer requests suboffsets, the exporter is allowed to supply null.
+ // In theory, the exporter could require that it be requested and still supply null.
+ }
+
+ /**
+ * Provide an instance of <code>SimpleReadonlyBuffer</code> in a default, semi-constructed
+ * state. The sub-class constructor takes responsibility for completing construction including a
+ * call to {@link #assignCapabilityFlags(int, int, int, int)}.
+ *
+ * @param exporter the exporting object
+ * @param buf wrapping the array of bytes storing the implementation of the object
+ */
+ protected SimpleReadonlyBuffer(BufferProtocol exporter, BufferPointer buf) {
+ super(exporter, buf);
+ }
+
+ /**
+ * Provide an instance of SimpleReadonlyBuffer meeting the consumer's expectations as expressed
+ * in the flags argument.
+ *
+ * @param exporter the exporting object
+ * @param buf wrapping the array of bytes storing the implementation of the object
+ * @param flags consumer requirements
+ */
+ public SimpleReadonlyBuffer(BufferProtocol exporter, BufferPointer buf, int flags) {
+ super(exporter, buf);
+ assignCapabilityFlags(flags, REQUIRED_FLAGS, ALLOWED_FLAGS, IMPLIED_FLAGS);
+ fillInfo();
+ }
+
+ @Override
+ public int getNdim() {
+ return 1;
+ }
+
+ @Override
+ public byte byteAt(int index) throws IndexOutOfBoundsException {
+ // offset is not necessarily zero
+ return buf.storage[buf.offset + index];
+ }
+
+ @Override
+ public int intAt(int index) throws IndexOutOfBoundsException {
+ // Implement directly: a bit quicker than the default
+ return 0xff & buf.storage[buf.offset + index];
+ }
+
+ @Override
+ public byte byteAt(int... indices) throws IndexOutOfBoundsException {
+ if (indices.length != 1) {
+ checkDimension(indices);
+ }
+ return byteAt(indices[0]);
+ }
+
+ @Override
+ public void copyTo(int srcIndex, byte[] dest, int destPos, int length)
+ throws IndexOutOfBoundsException {
+ System.arraycopy(buf.storage, buf.offset + srcIndex, dest, destPos, length);
+ }
+
+ @Override
+ public BufferPointer getPointer(int index) {
+ return new BufferPointer(buf.storage, buf.offset + index, 1);
+ }
+
+ @Override
+ public BufferPointer getPointer(int... indices) {
+ if (indices.length != 1) {
+ checkDimension(indices);
+ }
+ return getPointer(indices[0]);
+ }
+
+}
diff --git a/src/org/python/core/buffer/SimpleStringBuffer.java b/src/org/python/core/buffer/SimpleStringBuffer.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/core/buffer/SimpleStringBuffer.java
@@ -0,0 +1,149 @@
+package org.python.core.buffer;
+
+import org.python.core.BufferPointer;
+import org.python.core.BufferProtocol;
+import org.python.core.util.StringUtil;
+
+/**
+ * Buffer API that appears to be a one-dimensional array of one-byte items providing read-only API,
+ * but which is actually backed by a Java String. Some of the buffer API absolutely needs access to
+ * the data as a byte array (those parts that involve a {@link BufferPointer} result), and therefore
+ * this class must create a byte array from the String for them. However, it defers creation of a
+ * byte array until that part of the API is actually used. This class overrides those methods in
+ * SimpleReadonlyBuffer that would access the <code>buf</code> attribute to work out their results
+ * from the String instead.
+ */
+public class SimpleStringBuffer extends SimpleReadonlyBuffer {
+
+ /**
+ * The string backing this PyBuffer. A substitute for {@link #buf} until we can no longer avoid
+ * creating it.
+ */
+ private String bufString;
+
+ /**
+ * Partial counterpart to CPython <code>PyBuffer_FillInfo()</code> specific to the simple type
+ * of buffer and called from the constructor. The base constructor will already have been
+ * called, filling {@link #bufString} and {@link #obj}. And the method
+ * {@link #assignCapabilityFlags(int, int, int, int)} has set {@link #capabilityFlags}.
+ */
+ protected void fillInfo(String bufString) {
+ /*
+ * We will already have called: assignCapabilityFlags(flags, requiredFlags, allowedFlags,
+ * impliedFlags); So capabilityFlags holds the requests for shape, strides, writable, etc..
+ */
+ // Save the backing string
+ this.bufString = bufString;
+
+ // Difference from CPython: never null, even when the consumer doesn't request it
+ shape = new int[1];
+ shape[0] = bufString.length();
+
+ // Following CPython: provide strides only when the consumer requests it
+ if ((capabilityFlags & STRIDES) == STRIDES) {
+ strides = SIMPLE_STRIDES;
+ }
+
+ // Even when the consumer requests suboffsets, the exporter is allowed to supply null.
+ // In theory, the exporter could require that it be requested and still supply null.
+ }
+
+ /**
+ * Provide an instance of SimpleReadonlyBuffer meeting the consumer's expectations as expressed
+ * in the flags argument.
+ *
+ * @param exporter the exporting object
+ * @param bufString storing the implementation of the object
+ * @param flags consumer requirements
+ */
+ public SimpleStringBuffer(BufferProtocol exporter, String bufString, int flags) {
+ super(exporter, null);
+ assignCapabilityFlags(flags, REQUIRED_FLAGS, ALLOWED_FLAGS, IMPLIED_FLAGS);
+ fillInfo(bufString);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * This method uses {@link String#length()} rather than create an actual byte buffer.
+ */
+ @Override
+ public int getLen() {
+ // Avoid creating buf by using String.length
+ return bufString.length();
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * This method uses {@link String#charAt(int)} rather than create an actual byte buffer.
+ */
+ @Override
+ public byte byteAt(int index) throws IndexOutOfBoundsException {
+ // Avoid creating buf by using String.charAt
+ return (byte)bufString.charAt(index);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * This method uses {@link String#charAt(int)} rather than create an actual byte buffer.
+ */
+ @Override
+ public int intAt(int index) throws IndexOutOfBoundsException {
+ // Avoid creating buf by using String.charAt
+ return bufString.charAt(index);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * This method uses {@link String#charAt(int)} rather than create an actual byte buffer.
+ */
+ @Override
+ public void copyTo(int srcIndex, byte[] dest, int destPos, int length)
+ throws IndexOutOfBoundsException {
+ // Avoid creating buf by using String.charAt
+ int endIndex = srcIndex + length, p = destPos;
+ for (int i = srcIndex; i < endIndex; i++) {
+ dest[p++] = (byte)bufString.charAt(i);
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * This method creates an actual byte buffer from the String if none yet exists.
+ */
+ @Override
+ public BufferPointer getBuf() {
+ if (buf == null) {
+ // We can't avoid creating buf any longer
+ buf = new BufferPointer(StringUtil.toBytes(bufString));
+ }
+ return buf;
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * This method creates an actual byte buffer from the String if none yet exists.
+ */
+ @Override
+ public BufferPointer getPointer(int index) {
+ getBuf(); // Ensure buffer created
+ return super.getPointer(index);
+ }
+
+ /**
+ * {@inheritDoc}
+ * <p>
+ * This method creates an actual byte buffer from the String if none yet exists.
+ */
+ @Override
+ public BufferPointer getPointer(int... indices) {
+ getBuf(); // Ensure buffer created
+ return super.getPointer(indices);
+ }
+
+}
diff --git a/tests/java/org/python/core/BaseBytesTest.java b/tests/java/org/python/core/BaseBytesTest.java
--- a/tests/java/org/python/core/BaseBytesTest.java
+++ b/tests/java/org/python/core/BaseBytesTest.java
@@ -4,10 +4,11 @@
import java.util.List;
import java.util.Random;
+import junit.framework.TestCase;
+
+import org.python.core.buffer.SimpleBuffer;
import org.python.util.PythonInterpreter;
-import junit.framework.TestCase;
-
/**
* Unit test of org.python.core.BaseBytes, a class that supplies much of the behaviour of the Jython
* bytearray. In fact, it supplies almost all the immutable behaviour, and is abstract. In order to
@@ -517,7 +518,7 @@
return new MyBytes(value);
}
- public BaseBytes getInstance(MemoryViewProtocol value) throws PyException {
+ public BaseBytes getInstance(BufferProtocol value) throws PyException {
return new MyBytes(value);
}
@@ -599,9 +600,9 @@
*
* @param value source of the bytes (and size)
*/
- public MyBytes(MemoryViewProtocol value) {
+ public MyBytes(BufferProtocol value) {
super(TYPE);
- init(value.getMemoryView());
+ init((BufferProtocol)value.getBuffer(PyBUF.SIMPLE));
}
/**
@@ -769,13 +770,12 @@
/**
* An object that for test purposes (of construction and slice assignment) contains an array of
- * values that it is able to offer for reading through the MemoryView interface.
+ * values that it is able to offer for reading through the PyBuffer interface.
*/
- public static class MemoryViewable extends PyObject implements MemoryViewProtocol {
+ public static class BufferedObject extends PyObject implements BufferProtocol {
- public static final PyType TYPE = PyType.fromClass(MemoryViewable.class);
+ public static final PyType TYPE = PyType.fromClass(BufferedObject.class);
- private MemoryView mv;
private byte[] store;
/**
@@ -783,7 +783,7 @@
*
* @param value integers to store
*/
- MemoryViewable(int[] value) {
+ BufferedObject(int[] value) {
super(TYPE);
int n = value.length;
store = new byte[n];
@@ -793,56 +793,10 @@
}
@Override
- public MemoryView getMemoryView() {
- if (mv == null) {
- mv = new MemoryViewImpl();
- }
- return mv;
+ public PyBuffer getBuffer(int flags) {
+ return new SimpleBuffer(this, new BufferPointer(store), flags);
}
- /**
- * All instances of MemoryViewable have one dimension with stride one.
- */
- private static final PyTuple STRIDES = new PyTuple(Py.One);
-
- /**
- * Very simple MemoryView for one-dimensional byte array.
- */
- class MemoryViewImpl implements MemoryView {
-
- private final PyTuple shape = new PyTuple(new PyInteger(store.length));
-
- @Override
- public String get_format() {
- return "B";
- }
-
- @Override
- public int get_itemsize() {
- return 1;
- }
-
- @Override
- public PyTuple get_shape() {
- return shape;
- }
-
- @Override
- public int get_ndim() {
- return 1;
- }
-
- @Override
- public PyTuple get_strides() {
- return STRIDES;
- }
-
- @Override
- public boolean get_readonly() {
- return true;
- }
-
- }
}
/**
diff --git a/tests/java/org/python/core/PyBufferTest.java b/tests/java/org/python/core/PyBufferTest.java
new file mode 100644
--- /dev/null
+++ b/tests/java/org/python/core/PyBufferTest.java
@@ -0,0 +1,1094 @@
+package org.python.core;
+
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+
+import junit.framework.TestCase;
+
+import org.python.core.buffer.SimpleBuffer;
+import org.python.core.buffer.SimpleReadonlyBuffer;
+import org.python.core.buffer.SimpleStringBuffer;
+import org.python.util.PythonInterpreter;
+
+/**
+ * Test the several implementations (and exporters) of the PyBuffer interface provided in the Jython
+ * core.
+ * <p>
+ * The approach is to create test material once that has the necessary variety in byte array values,
+ * then for each test, when the JUnit framework creates an instance of the function-specific test,
+ * to use this material to create instances of each read-only type and each writable type. Writable
+ * instance types go onto the lists buffersToRead and buffersToWrite, while read-only instances go
+ * onto the lists buffersToRead and buffersToFailToWrite.
+ * <p>
+ * In general, tests of methods that read data apply themselves to all the elements of the
+ * buffersToRead list, while tests of methods that write data apply themselves to all the elements
+ * of the buffersToWrite list and check that members of the buffersToFailToWrite list raise an
+ * exception.
+ * <p>
+ * The Jython buffer API follows the structures of the CPython buffer API so that it supports in
+ * principle the use of multi-dimensional, strided add indirect array structures as buffers.
+ * However, actual buffers in the Jython core, and therefore these tests, limit themselves to one
+ * dimensional contiguous buffers with a simple organisation. Some tests apply directly to the
+ * N-dimensional cases, and some need a complete re-think. Sub-classing this test would probably be
+ * a good way to extend it to a wider range.
+ */
+public class PyBufferTest extends TestCase {
+
+ /**
+ * Generated constructor
+ *
+ * @param name
+ */
+ public PyBufferTest(String name) {
+ super(name);
+ }
+
+ /** Sometimes we need the interpreter to be initialised **/
+ PythonInterpreter interp;
+
+ /*
+ * Values for initialising the exporters.
+ */
+ private static final ByteMaterial byteMaterial = new ByteMaterial(0, 17, 16);
+ private static final ByteMaterial abcMaterial = new ByteMaterial("abcdef");
+ private static final ByteMaterial stringMaterial = new ByteMaterial("Mon côté fâcheux");
+ private static final ByteMaterial emptyMaterial = new ByteMaterial(new byte[0]);
+ private static final ByteMaterial longMaterial = new ByteMaterial(0, 5, 1000);
+
+ protected void setUp() throws Exception {
+ super.setUp();
+
+ // Exception raising requires the Jython interpreter
+ interp = new PythonInterpreter();
+
+ // Tests using local examples
+ queueWrite(new SimpleExporter(abcMaterial.getBytes()), abcMaterial);
+ queueReadonly(new SimpleExporter(byteMaterial.getBytes(), true), byteMaterial);
+ queueReadonly(new StringExporter(stringMaterial.string), stringMaterial);
+ queueWrite(new SimpleExporter(emptyMaterial.getBytes()), emptyMaterial);
+
+ // Tests with PyByteArray
+ queueWrite(new PyByteArray(abcMaterial.getBytes()), abcMaterial);
+ queueWrite(new PyByteArray(longMaterial.getBytes()), longMaterial);
+ queueWrite(new PyByteArray(), emptyMaterial);
+
+ // Tests with PyString
+ queueReadonly(new PyString(abcMaterial.string), abcMaterial);
+ queueReadonly(new PyString(), emptyMaterial);
+
+ // Ensure case is tested where PyByteArray has an internal offset
+ PyByteArray truncated = new PyByteArray(stringMaterial.getBytes());
+ truncated.delRange(0, 4);
+ ByteMaterial truncatedMaterial = new ByteMaterial(stringMaterial.string.substring(4));
+ assert truncated.__alloc__() > truncatedMaterial.length;
+ queueWrite(truncated, truncatedMaterial);
+ }
+
+ private void queueWrite(BufferProtocol exporter, ByteMaterial material) {
+ BufferTestPair pair = new BufferTestPair(exporter, material);
+ buffersToRead.add(pair);
+ buffersToWrite.add(pair);
+ }
+
+ private void queueReadonly(BufferProtocol exporter, ByteMaterial material) {
+ BufferTestPair pair = new BufferTestPair(exporter, material);
+ buffersToRead.add(pair);
+ buffersToFailToWrite.add(pair);
+ }
+
+ /** Read operations should succeed on all these objects. */
+ private List<BufferTestPair> buffersToRead = new LinkedList<BufferTestPair>();
+ /** Write operations should succeed on all these objects. */
+ private List<BufferTestPair> buffersToWrite = new LinkedList<BufferTestPair>();
+ /** Write operations should fail on all these objects. */
+ private List<BufferTestPair> buffersToFailToWrite = new LinkedList<BufferTestPair>();
+
+ /** We should be able to get a buffer for all these flag types. */
+ private int[] validFlags = {PyBUF.SIMPLE, PyBUF.ND, PyBUF.STRIDES, PyBUF.INDIRECT};
+
+ /** To which we can add any of these (in one dimension, anyway) */
+ private int[] validTassles = {PyBUF.FORMAT,
+ PyBUF.C_CONTIGUOUS,
+ PyBUF.F_CONTIGUOUS,
+ PyBUF.ANY_CONTIGUOUS};
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#getBuf()}.
+ */
+ public void testGetBuffer() {
+
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("getBuffer(): " + test);
+ for (int flags : validFlags) {
+ for (int tassle : validTassles) {
+ PyBuffer view = test.exporter.getBuffer(flags | tassle);
+ assertNotNull(view);
+ }
+ }
+ }
+
+ for (BufferTestPair test : buffersToWrite) {
+ System.out.println("getBuffer(WRITABLE): " + test);
+ for (int flags : validFlags) {
+ for (int tassle : validTassles) {
+ PyBuffer view = test.exporter.getBuffer(flags | tassle | PyBUF.WRITABLE);
+ assertNotNull(view);
+ }
+ }
+ }
+
+ for (BufferTestPair test : buffersToFailToWrite) {
+ System.out.println("getBuffer(WRITABLE): " + test);
+ for (int flags : validFlags) {
+ try {
+ test.exporter.getBuffer(flags | PyBUF.WRITABLE);
+ fail("Write access not prevented: " + test);
+ } catch (PyException pye) {
+ // Expect BufferError
+ assertEquals(Py.BufferError, pye.type);
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBUF#isReadonly()}.
+ */
+ public void testIsReadonly() {
+
+ for (BufferTestPair test : buffersToWrite) {
+ System.out.println("isReadonly: " + test);
+ assertFalse(test.simple.isReadonly());
+ }
+
+ for (BufferTestPair test : buffersToFailToWrite) {
+ System.out.println("isReadonly: " + test);
+ assertTrue(test.simple.isReadonly());
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBUF#getNdim()}.
+ */
+ public void testGetNdim() {
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("getNdim: " + test);
+ assertEquals("simple ndim", test.shape.length, test.simple.getNdim());
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBUF#getShape()}.
+ */
+ public void testGetShape() {
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("getShape: " + test);
+ int[] shape = test.simple.getShape();
+ assertNotNull(shape);
+ assertIntsEqual("simple shape", test.shape, shape);
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBUF#getLen()}.
+ */
+ public void testGetLen() {
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("getLen: " + test);
+ assertEquals(" simple len", test.material.bytes.length, test.simple.getLen());
+ assertEquals("strided len", test.material.bytes.length, test.strided.getLen());
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#byteAt(int)}.
+ */
+ public void testByteAt() {
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("byteAt: " + test);
+ int n = test.material.length;
+ byte[] exp = test.material.bytes;
+ for (int i = 0; i < n; i++) {
+ assertEquals(exp[i], test.simple.byteAt(i));
+ }
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#byteAt(int[])}.
+ */
+ public void testByteAtNdim() {
+ int[] index = new int[1];
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("byteAt(array): " + test);
+ if (test.strided.getShape().length != 1) {
+ fail("Test not implemented dimensions != 1");
+ }
+ byte[] exp = test.material.bytes;
+ int n = test.material.length;
+ // Run through 1D index for simple
+ for (int i = 0; i < n; i++) {
+ index[0] = i;
+ assertEquals(exp[i], test.simple.byteAt(index));
+ }
+ // Check 2D index throws
+ try {
+ test.simple.byteAt(0, 0);
+ fail("Use of 2D index did not raise exception");
+ } catch (PyException pye) {
+ // Expect BufferError
+ assertEquals(Py.BufferError, pye.type);
+ }
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#intAt(int)}.
+ */
+ public void testIntAt() {
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("intAt: " + test);
+ int n = test.material.length;
+ int[] exp = test.material.ints;
+ for (int i = 0; i < n; i++) {
+ assertEquals(exp[i], test.simple.intAt(i));
+ }
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#intAt(int[])}.
+ */
+ public void testIntAtNdim() {
+ int[] index = new int[1];
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("intAt(array): " + test);
+ if (test.strided.getShape().length != 1) {
+ fail("Test not implemented dimensions != 1");
+ }
+ int[] exp = test.material.ints;
+ int n = test.material.length;
+ // Run through 1D index for simple
+ for (int i = 0; i < n; i++) {
+ index[0] = i;
+ assertEquals(exp[i], test.simple.intAt(index));
+ }
+ // Check 2D index throws
+ try {
+ test.simple.intAt(0, 0);
+ fail("Use of 2D index did not raise exception");
+ } catch (PyException pye) {
+ // Expect BufferError
+ assertEquals(Py.BufferError, pye.type);
+ }
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#storeAt(byte, int)}.
+ */
+ public void testStoreAt() {
+ for (BufferTestPair test : buffersToWrite) {
+ System.out.println("storeAt: " + test);
+ int n = test.material.length;
+ int[] exp = test.material.ints;
+ // Write modified test material into each location using storeAt()
+ for (int i = 0; i < n; i++) {
+ byte v = (byte)(exp[i] ^ 3); // twiddle some bits
+ test.simple.storeAt(v, i);
+ }
+ // Compare each location with modified test data using intAt()
+ for (int i = 0; i < n; i++) {
+ assertEquals(exp[i] ^ 3, test.simple.intAt(i));
+ }
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#storeAt(byte, int[])}.
+ */
+ public void testStoreAtNdim() {
+ for (BufferTestPair test : buffersToWrite) {
+ System.out.println("storeAt: " + test);
+ int n = test.material.length;
+ int[] exp = test.material.ints;
+ // Write modified test material into each location using storeAt()
+ for (int i = 0; i < n; i++) {
+ byte v = (byte)(exp[i] ^ 3); // twiddle some bits
+ test.simple.storeAt(v, i);
+ }
+ // Compare each location with modified test data using intAt()
+ for (int i = 0; i < n; i++) {
+ assertEquals(exp[i] ^ 3, test.simple.intAt(i));
+ }
+ // Check 2D index throws
+ try {
+ test.simple.storeAt((byte)1, 0, 0);
+ fail("Use of 2D index did not raise exception");
+ } catch (PyException pye) {
+ // Expect BufferError
+ assertEquals(Py.BufferError, pye.type);
+ }
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#copyTo(byte[], int)}.
+ */
+ public void testCopyTo() {
+ final int OFFSET = 5;
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("copyTo: " + test);
+ int n = test.material.length;
+ // Try with zero offset
+ byte[] actual = new byte[n];
+ test.simple.copyTo(actual, 0);
+ assertBytesEqual("copyTo() incorrect", test.material.bytes, actual, 0);
+ // Try to middle of array
+ actual = new byte[n + 2 * OFFSET];
+ test.simple.copyTo(actual, OFFSET);
+ assertBytesEqual("copyTo(offset) incorrect", test.material.bytes, actual, OFFSET);
+ assertEquals("data before destination", 0, actual[OFFSET - 1]);
+ assertEquals("data after destination", 0, actual[OFFSET + n]);
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#copyTo(int, byte[], int, int)}.
+ */
+ public void testSliceCopyTo() {
+ final int OFFSET = 5;
+ final byte BLANK = 7;
+
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("copyTo(from slice): " + test);
+ PyBuffer view = test.simple;
+
+ int n = test.material.length;
+ byte[] actual = new byte[n + 2 * OFFSET];
+
+ // Try destination positions in actual[] of 0 and OFFSET
+ for (int destPos = 0; destPos <= OFFSET; destPos += OFFSET) {
+ // Try source positions in 0 and OFFSET
+ for (int srcIndex = 0; srcIndex <= OFFSET; srcIndex += OFFSET) {
+
+ // A variety of lengths from zero to (n-srcIndex)-ish
+ for (int length = 0; srcIndex + length <= n; length = 2 * length + 1) {
+ /*
+ * System.out.printf(" copy src[%d:%d] (%d) to dst[%d:%d] (%d)\n",
+ * srcIndex, srcIndex + length, n, destPos, destPos + length,
+ * actual.length);
+ */
+ Arrays.fill(actual, BLANK);
+
+ // Test the method
+ view.copyTo(srcIndex, actual, destPos, length);
+
+ // Check changed part of destination
+ assertBytesEqual("copyTo(slice) incorrect", test.material.bytes, srcIndex,
+ actual, destPos, length);
+ if (destPos > 0) {
+ assertEquals("data before destination", BLANK, actual[destPos - 1]);
+ }
+ assertEquals("data after destination", BLANK, actual[destPos + length]);
+ }
+
+ // And from exactly n-srcIndex down to zero-ish
+ for (int trim = 0; srcIndex + trim <= n; trim = 2 * trim + 1) {
+ int length = n - srcIndex - trim;
+ /*
+ * System.out.printf(" copy src[%d:%d] (%d) to dst[%d:%d] (%d)\n",
+ * srcIndex, srcIndex + length, n, destPos, destPos + length,
+ * actual.length);
+ */
+ Arrays.fill(actual, BLANK);
+
+ // Test the method
+ view.copyTo(srcIndex, actual, destPos, length);
+
+ // Check changed part of destination
+ assertBytesEqual("copyTo(slice) incorrect", test.material.bytes, srcIndex,
+ actual, destPos, length);
+ if (destPos > 0) {
+ assertEquals("data before destination", BLANK, actual[destPos - 1]);
+ }
+ assertEquals("data after destination", BLANK, actual[destPos + length]);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#copyFrom(byte[], int, int, int)}.
+ */
+ public void testCopyFrom() {
+ final int OFFSET = 5;
+ final byte BLANK = 7;
+
+ for (BufferTestPair test : buffersToWrite) {
+ System.out.println("copyFrom(): " + test);
+ PyBuffer view = test.simple;
+
+ int n = test.material.length;
+ byte[] actual = new byte[n];
+ byte[] expected = new byte[n];
+
+ // Make some source material for copies (need to test at OFFSET too).
+ byte[] src = new byte[n + OFFSET];
+ for (int i = 0; i < src.length; i++) {
+ src[i] = (byte)i;
+ }
+
+ // Try destination positions in test object of 0 and OFFSET
+ for (int destIndex = 0; destIndex <= OFFSET; destIndex += OFFSET) {
+
+ // Try source positions in 0 and OFFSET
+ for (int srcPos = 0; srcPos <= OFFSET; srcPos += OFFSET) {
+
+ // A variety of lengths from zero to (n-destIndex)-ish
+ for (int length = 0; destIndex + length <= n; length = 2 * length + 1) {
+
+ System.out.printf(" copy src[%d:%d] (%d) to dst[%d:%d] (%d)\n", srcPos,
+ srcPos + length, n, destIndex, destIndex + length,
+ actual.length);
+
+ // Initialise the object (have to do each time) and expected value
+ for (int i = 0; i < n; i++) {
+ expected[i] = BLANK;
+ view.storeAt(BLANK, i);
+ }
+
+ // Test the method and extract the result to actual[]
+ view.copyFrom(src, srcPos, destIndex, length);
+ view.copyTo(actual, 0);
+
+ // Complete what is should be in expected[]
+ for (int i = 0; i < length; i++) {
+ expected[destIndex + i] = src[srcPos + i];
+ }
+ assertBytesEqual("copyFrom() incorrect", expected, actual, 0);
+ }
+
+ // And from exactly n-destIndex down to zero-ish
+ for (int trim = 0; destIndex + trim <= n; trim = 2 * trim + 1) {
+ int length = n - destIndex - trim;
+ System.out.printf(" copy src[%d:%d] (%d) to dst[%d:%d] (%d)\n", srcPos,
+ srcPos + length, n, destIndex, destIndex + length,
+ actual.length);
+
+ // Initialise the object (have to do each time) and expected value
+ for (int i = 0; i < n; i++) {
+ expected[i] = BLANK;
+ view.storeAt(BLANK, i);
+ }
+
+ // Test the method and extract the result to actual[]
+ view.copyFrom(src, srcPos, destIndex, length);
+ view.copyTo(actual, 0);
+
+ // Complete what is should be in expected[]
+ for (int i = 0; i < length; i++) {
+ expected[destIndex + i] = src[srcPos + i];
+ }
+ assertBytesEqual("copyFrom() incorrect", expected, actual, 0);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#getBuf()}.
+ */
+ public void testGetBuf() {
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("getBuf: " + test);
+ PyBuffer view = test.exporter.getBuffer(PyBUF.SIMPLE);
+ ByteMaterial m = test.material;
+
+ BufferPointer bp = view.getBuf();
+ assertBytesEqual("getBuf: ", m.bytes, bp);
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#getPointer(int)}.
+ */
+ public void testGetPointer() {
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("getPointer: " + test);
+ PyBuffer view = test.strided;
+ int n = test.material.length, itemsize = view.getItemsize();
+ byte[] exp = new byte[itemsize], bytes = test.material.bytes;
+
+ for (int i = 0; i < n; i++) {
+ // Expected result is one item (allow for itemsize)
+ int p = i * itemsize;
+ for (int j = 0; j < itemsize; j++) {
+ exp[j] = bytes[p + j];
+ }
+
+ // Get pointer and check contents for correct data
+ BufferPointer bp = view.getPointer(i);
+ assertBytesEqual("getPointer value", exp, bp.storage, bp.offset);
+ assertEquals("getPointer size wrong", itemsize, bp.size);
+ }
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#getPointer(int[])}.
+ */
+ public void testGetPointerNdim() {
+ int[] index = new int[1];
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("getPointer(array): " + test);
+ PyBuffer view = test.strided;
+ int n = test.material.length, itemsize = view.getItemsize();
+ byte[] exp = new byte[itemsize], bytes = test.material.bytes;
+
+ for (int i = 0; i < n; i++) {
+ // Expected result is one item (allow for itemsize)
+ int p = i * itemsize;
+ for (int j = 0; j < itemsize; j++) {
+ exp[j] = bytes[p + j];
+ }
+
+ // Get pointer and check contents for correct data
+ index[0] = i;
+ BufferPointer bp = view.getPointer(index);
+ assertBytesEqual("getPointer value", exp, bp.storage, bp.offset);
+ assertEquals("getPointer size wrong", itemsize, bp.size);
+ }
+ // Check 2D index throws
+ try {
+ view.getPointer(0, 0);
+ fail("Use of 2D index did not raise exception");
+ } catch (PyException pye) {
+ // Expect BufferError
+ assertEquals(Py.BufferError, pye.type);
+ }
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBUF#release()}.
+ */
+ public void testRelease() {
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("release: " + test);
+ BufferProtocol obj = test.exporter;
+ // The object should already be exporting test.simple and test.strided
+ PyBuffer a = test.simple; // 1
+ PyBuffer b = test.strided; // 2
+ PyBuffer c = obj.getBuffer(PyBUF.SIMPLE | PyBUF.FORMAT); // 3
+ checkExporting(obj);
+ // Multiple releases of the same buffer are just one release
+ b.release(); // 2
+ b.release();
+ b.release();
+ b.release();
+ checkExporting(obj);
+ // Now see that releasing in some other order works correctly
+ a.release(); // 1
+ checkExporting(obj);
+ PyBuffer d = obj.getBuffer(PyBUF.STRIDES | PyBUF.FORMAT); // 2
+ c.release(); // 1
+ checkExporting(obj);
+ d.release(); // 0
+ checkNotExporting(obj);
+ d.release(); // 0
+ checkNotExporting(obj);
+ }
+ }
+
+ /**
+ * Error if exporter is not actually exporting (and is of a type that locks on export).
+ *
+ * @param exporter
+ */
+ private void checkExporting(BufferProtocol exporter) {
+ if (exporter instanceof SimpleExporter) {
+ assertTrue("exports not being counted", ((SimpleExporter)exporter).exportCount >= 1);
+ } else if (exporter instanceof PyByteArray) {
+ // Size-changing access should fail
+ try {
+ ((PyByteArray)exporter).bytearray_extend(Py.One);
+ fail("bytearray_extend with exports should fail");
+ } catch (Exception e) {
+ // Success
+ }
+ }
+ // Other types cannot be checked
+ }
+
+ /**
+ * Error if exporter is exporting (and is of a type that locks on export).
+ *
+ * @param exporter
+ */
+ private void checkNotExporting(BufferProtocol exporter) {
+ if (exporter instanceof SimpleExporter) {
+ assertFalse("exports falsely counted", ((SimpleExporter)exporter).exportCount >= 1);
+ } else if (exporter instanceof PyByteArray) {
+ // Size-changing access should fail
+ try {
+ ((PyByteArray)exporter).bytearray_extend(Py.One);
+ } catch (Exception e) {
+ fail("bytearray unexpectedly locked");
+ }
+ }
+ // Other types cannot be checked
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBUF#getStrides()}.
+ */
+ public void testGetStrides() {
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("getStrides: " + test);
+ // When not requested ...
+ assertNull(test.simple.getStrides());
+ // When requested, ought to be as expected
+ int[] strides = test.strided.getStrides();
+ assertNotNull(strides);
+ assertIntsEqual("strided.strides", test.strides, strides);
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBUF#getSuboffsets()}.
+ */
+ public void testGetSuboffsets() {
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("getSuboffsets: " + test);
+ // Null for all test material
+ assertNull(test.simple.getSuboffsets());
+ assertNull(test.strided.getSuboffsets());
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBUF#isContiguous(char)}.
+ */
+ public void testIsContiguous() {
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("isContiguous: " + test);
+ // True for all test material and orders (since 1-dimensional)
+ for (char order : validOrders) {
+ assertTrue(test.simple.isContiguous(order));
+ assertTrue(test.strided.isContiguous(order));
+ }
+ }
+ }
+
+ private static final char[] validOrders = {'C', 'F', 'A'};
+
+ /**
+ * Test method for {@link org.python.core.PyBuffer#getFormat()}.
+ */
+ public void testGetFormat() {
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("getFormat: " + test);
+ // Null for all test material
+ assertNull(test.simple.getFormat());
+ assertNull(test.strided.getFormat());
+ // However, we can ask for it explicitly ...
+ PyBuffer simpleWithFormat = test.exporter.getBuffer(PyBUF.SIMPLE | PyBUF.FORMAT);
+ PyBuffer stridedWithFormat = test.exporter.getBuffer(PyBUF.STRIDES | PyBUF.FORMAT);
+ // "B" for all test material where requested in flags
+ assertEquals("B", simpleWithFormat.getFormat());
+ assertEquals("B", stridedWithFormat.getFormat());
+ }
+ }
+
+ /**
+ * Test method for {@link org.python.core.PyBUF#getItemsize()}.
+ */
+ public void testGetItemsize() {
+ for (BufferTestPair test : buffersToRead) {
+ System.out.println("getItemsize: " + test);
+ // Unity for all test material
+ assertEquals(1, test.simple.getItemsize());
+ assertEquals(1, test.strided.getItemsize());
+ }
+ }
+
+ /**
+ * A class to act as an exporter that uses the SimpleBuffer (or SimpleReadonlyBuffer). This
+ * permits testing abstracted from the Jython interpreter.
+ */
+ static class SimpleExporter implements BufferProtocol {
+
+ byte[] storage;
+ int exportCount;
+ boolean readonly;
+
+ /**
+ * Construct a simple exporter from the bytes supplied.
+ *
+ * @param storage
+ */
+ public SimpleExporter(byte[] storage) {
+ this.storage = storage;
+ }
+
+ /**
+ * Construct a simple exporter from the bytes supplied, optionally read-only.
+ *
+ * @param storage
+ * @param readonly
+ */
+ public SimpleExporter(byte[] storage, boolean readonly) {
+ this.storage = storage;
+ this.readonly = readonly;
+ }
+
+ @Override
+ public PyBuffer getBuffer(int flags) {
+ BufferPointer mb = new BufferPointer(storage);
+ exportCount++;
+ if (readonly) {
+ return new SimpleReadonlyBuffer(this, mb, flags) {
+
+ protected void releaseAction() {
+ --exportCount;
+ }
+ };
+ } else {
+ return new SimpleBuffer(this, mb, flags) {
+
+ protected void releaseAction() {
+ --exportCount;
+ }
+ };
+ }
+ }
+ }
+
+ /**
+ * A class to act as an exporter that uses the SimpleStringBuffer. This permits testing
+ * abstracted from the Jython interpreter.
+ */
+ static class StringExporter implements BufferProtocol {
+
+ String storage;
+ int exportCount;
+
+ /**
+ * Construct a simple exporter from the String supplied.
+ *
+ * @param s
+ */
+ public StringExporter(String s) {
+ storage = s;
+ }
+
+ @Override
+ public PyBuffer getBuffer(int flags) {
+ return new SimpleStringBuffer(this, storage, flags);
+ }
+ }
+
+ /**
+ * Class to hold test material representing the same sequence of values 0..255 in several
+ * different ways.
+ */
+ protected static class ByteMaterial {
+
+ final String string;
+ final byte[] bytes;
+ final int[] ints;
+ final int length;
+
+ /** Construct from String. */
+ public ByteMaterial(String s) {
+ string = s;
+ length = s.length();
+ bytes = new byte[length];
+ ints = new int[length];
+ for (int i = 0; i < length; i++) {
+ int x = s.charAt(i);
+ ints[i] = x;
+ bytes[i] = (byte)x;
+ }
+ }
+
+ /** Construct from byte array. */
+ public ByteMaterial(byte[] b) {
+ length = b.length;
+ StringBuilder buf = new StringBuilder(length);
+ bytes = new byte[length];
+ ints = new int[length];
+ for (int i = 0; i < length; i++) {
+ int x = 0xff & b[i];
+ ints[i] = x;
+ bytes[i] = (byte)x;
+ buf.appendCodePoint(x);
+ }
+ string = buf.toString();
+ }
+
+ /** Construct from int array. */
+ public ByteMaterial(int[] a) {
+ length = a.length;
+ StringBuilder buf = new StringBuilder(length);
+ bytes = new byte[length];
+ ints = new int[length];
+ for (int i = 0; i < length; i++) {
+ int x = a[i];
+ ints[i] = x;
+ bytes[i] = (byte)x;
+ buf.appendCodePoint(x);
+ }
+ string = buf.toString();
+ }
+
+ /** Construct from pattern on values (used modulo 256). */
+ public ByteMaterial(int start, int inc, int count) {
+ length = count;
+ StringBuilder buf = new StringBuilder(length);
+ bytes = new byte[length];
+ ints = new int[length];
+ int x = start;
+ for (int i = 0; i < length; i++) {
+ ints[i] = x;
+ bytes[i] = (byte)x;
+ buf.appendCodePoint(x);
+ x = (x + inc) & 0xff;
+ }
+ string = buf.toString();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder(100);
+ buf.append("byte[").append(length).append("]={ ");
+ for (int i = 0; i < length; i++) {
+ if (i > 0) {
+ buf.append(", ");
+ }
+ if (i >= 5) {
+ buf.append(" ...");
+ break;
+ } else {
+ buf.append(ints[i]);
+ }
+ }
+ buf.append(" }");
+ return buf.toString();
+ }
+
+ /**
+ * @return a copy of the bytes array (that the client is allowed to modify)
+ */
+ byte[] getBytes() {
+ return bytes.clone();
+ }
+ }
+
+ /**
+ * Customised assert method comparing a buffer pointer to a byte array, usually the one from
+ * ByteMaterial.
+ *
+ * @param message to issue on failure
+ * @param expected expected byte array
+ * @param bp result to test
+ */
+ void assertBytesEqual(String message, byte[] expected, BufferPointer bp) {
+ int size = bp.size;
+ if (size != expected.length) {
+ fail(message + " (size)");
+ } else {
+ int len = bp.storage.length;
+ if (bp.offset < 0 || bp.offset + size > len) {
+ fail(message + " (offset)");
+ } else {
+ // Should be safe to compare the bytes
+ int i = bp.offset, j;
+ for (j = 0; j < size; j++) {
+ if (bp.storage[i++] != expected[j]) {
+ break;
+ }
+ }
+ if (j < size) {
+ fail(message + " (byte at " + j + ")");
+ }
+ }
+ }
+ }
+
+ /**
+ * Customised assert method comparing a buffer pointer to a byte array, usually the one from
+ * ByteMaterial.
+ *
+ * @param expected expected byte array
+ * @param bp result to test
+ */
+ void assertBytesEqual(byte[] expected, BufferPointer bp) {
+ assertBytesEqual("", expected, bp);
+ }
+
+ /**
+ * Customised assert method comparing a byte arrays: values in the actual value starting at
+ * actual[actualStart] must match all those in expected[], and there must be enough of them.
+ *
+ * @param message to issue on failure
+ * @param expected expected byte array
+ * @param actual result to test
+ * @param actualStart where to start the comparison in actual
+ */
+ void assertBytesEqual(String message, byte[] expected, byte[] actual, int actualStart) {
+ assertBytesEqual(message, expected, 0, actual, actualStart, expected.length);
+ }
+
+ /**
+ * Customised assert method comparing a byte arrays: values starting at actual[actualStart] must
+ * those starting at actual[actualStart], for a distance of n bytes.
+ *
+ * @param message to issue on failure
+ * @param expected expected byte array
+ * @param expectedStart where to start the comparison in expected
+ * @param actual result to test
+ * @param actualStart where to start the comparison in actual
+ * @param n number of bytes to test
+ */
+ void assertBytesEqual(String message, byte[] expected, int expectedStart, byte[] actual,
+ int actualStart, int n) {
+ if (actualStart < 0 || expectedStart < 0) {
+ fail(message + " (start<0)");
+ } else if (actualStart + n > actual.length || expectedStart + n > expected.length) {
+ fail(message + " (too short)");
+ } else {
+ // Should be safe to compare the values
+ int i = actualStart, j, jLimit = expectedStart + n;
+ for (j = expectedStart; j < jLimit; j++) {
+ if (actual[i++] != expected[j]) {
+ break;
+ }
+ }
+ if (j < jLimit) {
+ System.out.println(" expected:"
+ + Arrays.toString(Arrays.copyOfRange(expected, expectedStart, expectedStart
+ + n)));
+ System.out.println(" actual:"
+ + Arrays.toString(Arrays.copyOfRange(actual, actualStart, actualStart + n)));
+ System.out.println(" _actual_:" + Arrays.toString(actual));
+ fail(message + " (byte at " + j + ")");
+ }
+ }
+ }
+
+ /**
+ * Customised assert method comparing a int arrays: values in the actual value starting at
+ * actual[offset] must match all those in expected[], and there must be enough of them.
+ *
+ * @param message to issue on failure
+ * @param expected expected array
+ * @param actual result to test
+ * @param offset where to start the comparison in actual
+ */
+ void assertIntsEqual(String message, int[] expected, int[] actual, int offset) {
+ int n = expected.length;
+ if (offset < 0) {
+ fail(message + " (offset<0)");
+ } else if (offset + n > actual.length) {
+ fail(message + " (too short)");
+ } else {
+ // Should be safe to compare the values
+ int i = offset, j;
+ for (j = 0; j < n; j++) {
+ if (actual[i++] != expected[j]) {
+ break;
+ }
+ }
+ if (j < n) {
+ System.out.println(" expected:" + Arrays.toString(expected));
+ System.out.println(" actual:" + Arrays.toString(actual));
+ fail(message + " (int at " + j + ")");
+ }
+ }
+ }
+
+ /**
+ * Customised assert method comparing a int arrays: int in the actual value must match all those
+ * in expected[], and there must be the same number of them.
+ *
+ * @param message to issue on failure
+ * @param expected expected array
+ * @param actual result to test
+ */
+ void assertIntsEqual(String message, int[] expected, int[] actual) {
+ int n = expected.length;
+ assertEquals(message, n, actual.length);
+ // Should be safe to compare the values
+ int j;
+ for (j = 0; j < n; j++) {
+ if (actual[j] != expected[j]) {
+ break;
+ }
+ }
+ if (j < n) {
+ System.out.println(" expected:" + Arrays.toString(expected));
+ System.out.println(" actual:" + Arrays.toString(actual));
+ fail(message + " (int at " + j + ")");
+ }
+ }
+
+ /**
+ * Element for queueing tests, wraps an exporter object with (a copy of) the material from which
+ * it was created, and several PyBuffer views.
+ */
+ static class BufferTestPair {
+
+ static final int[] STRIDES_1D = {1};
+
+ BufferProtocol exporter;
+ ByteMaterial material;
+ PyBuffer simple, strided;
+ int[] shape, strides;
+
+ /**
+ * A test to do and the material for constructing it (and its results).
+ *
+ * @param exporter
+ * @param material
+ * @param shape of the array, when testing in N-dimensions
+ * @param stride of the array, when testing in N-dimensions
+ */
+ public BufferTestPair(BufferProtocol exporter, ByteMaterial material, int[] shape,
+ int[] strides) {
+ this.exporter = exporter;
+ this.material = new ByteMaterial(material.ints);
+ this.shape = shape;
+ this.strides = strides;
+ try {
+ simple = exporter.getBuffer(PyBUF.SIMPLE);
+ strided = exporter.getBuffer(PyBUF.STRIDES);
+ } catch (Exception e) {
+ // Leave them null if we can't get a PyBuffer: test being set up will fail.
+ // Silent here, but explicit test of getBuffer will reproduce and log this failure.
+ }
+ }
+
+ /**
+ * A test to do and the material for constructing it (and its results) in one dimension.
+ *
+ * @param exporter
+ * @param material
+ */
+ public BufferTestPair(BufferProtocol exporter, ByteMaterial material) {
+ this(exporter, material, new int[1], STRIDES_1D);
+ shape[0] = material.length;
+ }
+
+ @Override
+ public String toString() {
+ return exporter.getClass().getSimpleName() + "( " + material.toString() + " )";
+ }
+
+ }
+}
--
Repository URL: http://hg.python.org/jython
More information about the Jython-checkins
mailing list