From jython-checkins at python.org  Thu Mar  7 18:10:43 2019
From: jython-checkins at python.org (jeff.allen)
Date: Thu, 07 Mar 2019 23:10:43 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_=5BTrivial=5D_Fix_line_end?=
 =?utf-8?q?ings_in_JavaImportHelper=2E?=
Message-ID: <20190307231043.1.BDE8B563D9B9DAA2@mg.python.org>

https://hg.python.org/jython/rev/f956b03173ec
changeset:   8219:f956b03173ec
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Thu Jan 24 08:18:30 2019 +0000
summary:
  [Trivial] Fix line endings in JavaImportHelper.

files:
  src/org/python/core/JavaImportHelper.java |  440 +++++-----
  1 files changed, 220 insertions(+), 220 deletions(-)


diff --git a/src/org/python/core/JavaImportHelper.java b/src/org/python/core/JavaImportHelper.java
--- a/src/org/python/core/JavaImportHelper.java
+++ b/src/org/python/core/JavaImportHelper.java
@@ -1,220 +1,220 @@
-package org.python.core;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- * Helper class handling the VM specific java package detection.
- */
-public class JavaImportHelper {
-
-    private static final String DOT = ".";
-
-    /**
-     * Try to add the java package.
-     * <p>
-     * This is handy in cases where the package scan cannot run, or when the initial classpath does not contain all .jar
-     * files (such as in J2EE containers).
-     * <p>
-     * There is some self-healing in the sense that a correct, explicit import of a java class will succeed even if
-     * sys.modules already contains a Py.None entry for the corresponding java package.
-     * 
-     * @param packageName The dotted name of the java package
-     * @param fromlist A tuple with the from names to import. Can be null or empty.
-     * 
-     * @return <code>true</code> if a java package was doubtlessly identified and added, <code>false</code>
-     * otherwise.
-     */
-    protected static boolean tryAddPackage(final String packageName, PyObject fromlist) {
-        // make sure we do not turn off the added flag, once it is set
-        boolean packageAdded = false;
-
-        if (packageName != null) {
-            // check explicit imports first (performance optimization)
-
-            // handle 'from java.net import URL' like explicit imports
-            List<String> stringFromlist = getFromListAsStrings(fromlist);
-            for (String fromName : stringFromlist) {
-                if (isJavaClass(packageName, fromName)) {
-                    packageAdded = addPackage(packageName, packageAdded);
-
-                }
-            }
-
-            // handle 'import java.net.URL' style explicit imports
-            int dotPos = packageName.lastIndexOf(DOT);
-            if (dotPos > 0) {
-                String lastDottedName = packageName.substring(dotPos + 1);
-                String packageCand = packageName.substring(0, dotPos);
-                if (isJavaClass(packageCand, lastDottedName)) {
-                    packageAdded = addPackage(packageCand, packageAdded);
-                }
-            }
-
-            // if all else fails, check already loaded packages
-            if (!packageAdded) {
-                // build the actual map with the packages known to the VM
-                Map<String, String> packages = buildLoadedPackages();
-
-                // add known packages
-                String parentPackageName = packageName;
-                if (isLoadedPackage(packageName, packages)) {
-                    packageAdded = addPackage(packageName, packageAdded);
-                }
-                dotPos = 0;
-                do {
-                    dotPos = parentPackageName.lastIndexOf(DOT);
-                    if (dotPos > 0) {
-                        parentPackageName = parentPackageName.substring(0, dotPos);
-                        if (isLoadedPackage(parentPackageName, packages)) {
-                            packageAdded = addPackage(parentPackageName, packageAdded);
-                        }
-                    }
-                } while (dotPos > 0);
-
-                // handle package imports like 'from java import math'
-                for (String fromName : stringFromlist) {
-                    String fromPackageName = packageName + DOT + fromName;
-                    if (isLoadedPackage(fromPackageName, packages)) {
-                        packageAdded = addPackage(fromPackageName, packageAdded);
-                    }
-                }
-            }
-        }
-        return packageAdded;
-    }
-
-    /**
-     * Check if a java package is already known to the VM.
-     * <p>
-     * May return <code>false</code> even if the given package name is a valid java package !
-     * 
-     * @param packageName
-     * 
-     * @return <code>true</code> if the package with the given name is already loaded by the VM, <code>false</code>
-     * otherwise.
-     */
-    protected static boolean isLoadedPackage(String packageName) {
-        return isLoadedPackage(packageName, buildLoadedPackages());
-    }
-
-    /**
-     * Convert the fromlist into a java.lang.String based list.
-     * <p>
-     * Do some sanity checks: filter out '*' and empty tuples, as well as non tuples.
-     * 
-     * @param fromlist
-     * @return a list containing java.lang.String entries
-     */
-    private static final List<String> getFromListAsStrings(PyObject fromlist) {
-        List<String> stringFromlist = new ArrayList<String>();
-
-        if (fromlist != null && fromlist != Py.EmptyTuple && fromlist instanceof PyTuple) {
-            Iterator iterator = ((PyTuple) fromlist).iterator();
-            while (iterator.hasNext()) {
-                Object obj = iterator.next();
-                if (obj instanceof PyString) {
-                    obj = ((PyString) obj).getString();
-                }
-                if (obj instanceof String) {
-                    String fromName = (String) obj;
-                    if (!"*".equals(fromName)) {
-                        stringFromlist.add(fromName);
-                    }
-                }
-            }
-        }
-        return stringFromlist;
-    }
-
-    /**
-     * Faster way to check if a java package is already known to the VM.
-     * <p>
-     * May return <code>false</code> even if the given package name is a valid java package !
-     * 
-     * @param packageName
-     * @param packages A Map containing all packages actually known to the VM. Such a Map can be obtained using
-     * {@link JavaImportHelper.buildLoadedPackagesTree()}
-     * 
-     * @return <code>true</code> if the package with the given name is already loaded by the VM, <code>false</code>
-     * otherwise.
-     */
-    private static boolean isLoadedPackage(String javaPackageName, Map<String, String> packages) {
-        boolean isLoaded = false;
-        if (javaPackageName != null) {
-            isLoaded = packages.containsKey(javaPackageName);
-        }
-        return isLoaded;
-    }
-
-    /**
-     * Build a <code>Map</code> of the currently known packages to the VM.
-     * <p>
-     * All parent packages appear as single entries like python modules, e.g. <code>java</code>,
-     * <code>java.lang</code>, <code>java.lang.reflect</code>,
-     */
-    private static Map<String, String> buildLoadedPackages() {
-        TreeMap<String, String> packageMap = new TreeMap<String, String>();
-        Package[] packages = Package.getPackages();
-        for (int i = 0; i < packages.length; i++) {
-            String packageName = packages[i].getName();
-            packageMap.put(packageName, "");
-            int dotPos = 0;
-            do {
-                dotPos = packageName.lastIndexOf(DOT);
-                if (dotPos > 0) {
-                    packageName = packageName.substring(0, dotPos);
-                    packageMap.put(packageName, "");
-                }
-            } while (dotPos > 0);
-        }
-        return packageMap;
-    }
-
-    /**
-     * @return <code>true</code> if the java class can be found by the current
-     *         Py classloader setup
-     */
-    private static boolean isJavaClass(String packageName, String className) {
-        return className != null && className.length() > 0
-                && Py.findClass(packageName + "." + className) != null;
-    }
-
-    /**
-     * Add a java package to sys.modules, if not already done
-     * 
-     * @return <code>true</code> if something was really added, <code>false</code> otherwise
-     */
-    private static boolean addPackage(String packageName, boolean packageAdded) {
-        PyObject modules = Py.getSystemState().modules;
-        String internedPackageName = packageName.intern();
-        PyObject module = modules.__finditem__(internedPackageName);
-        // a previously failed import could have created a Py.None entry in sys.modules
-        if (module == null || module == Py.None) {
-            int dotPos = 0;
-            do {
-                PyJavaPackage p = PySystemState.add_package(packageName);
-                if(dotPos == 0) {
-                    modules.__setitem__(internedPackageName, p);
-                } else {
-                    module = modules.__finditem__(internedPackageName);
-                    if (module == null || module == Py.None) {
-                        modules.__setitem__(internedPackageName, p);
-                    }
-                }
-                dotPos = packageName.lastIndexOf(DOT);
-                if (dotPos > 0) {
-                    packageName = packageName.substring(0, dotPos);
-                    internedPackageName = packageName.intern();
-                }
-            } while(dotPos > 0);
-            // make sure not to turn off the packageAdded flag
-            packageAdded = true;
-        }
-        return packageAdded;
-    }
-
-}
+package org.python.core;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * Helper class handling the VM specific java package detection.
+ */
+public class JavaImportHelper {
+
+    private static final String DOT = ".";
+
+    /**
+     * Try to add the java package.
+     * <p>
+     * This is handy in cases where the package scan cannot run, or when the initial classpath does not contain all .jar
+     * files (such as in J2EE containers).
+     * <p>
+     * There is some self-healing in the sense that a correct, explicit import of a java class will succeed even if
+     * sys.modules already contains a Py.None entry for the corresponding java package.
+     *
+     * @param packageName The dotted name of the java package
+     * @param fromlist A tuple with the from names to import. Can be null or empty.
+     *
+     * @return <code>true</code> if a java package was doubtlessly identified and added, <code>false</code>
+     * otherwise.
+     */
+    protected static boolean tryAddPackage(final String packageName, PyObject fromlist) {
+        // make sure we do not turn off the added flag, once it is set
+        boolean packageAdded = false;
+
+        if (packageName != null) {
+            // check explicit imports first (performance optimization)
+
+            // handle 'from java.net import URL' like explicit imports
+            List<String> stringFromlist = getFromListAsStrings(fromlist);
+            for (String fromName : stringFromlist) {
+                if (isJavaClass(packageName, fromName)) {
+                    packageAdded = addPackage(packageName, packageAdded);
+
+                }
+            }
+
+            // handle 'import java.net.URL' style explicit imports
+            int dotPos = packageName.lastIndexOf(DOT);
+            if (dotPos > 0) {
+                String lastDottedName = packageName.substring(dotPos + 1);
+                String packageCand = packageName.substring(0, dotPos);
+                if (isJavaClass(packageCand, lastDottedName)) {
+                    packageAdded = addPackage(packageCand, packageAdded);
+                }
+            }
+
+            // if all else fails, check already loaded packages
+            if (!packageAdded) {
+                // build the actual map with the packages known to the VM
+                Map<String, String> packages = buildLoadedPackages();
+
+                // add known packages
+                String parentPackageName = packageName;
+                if (isLoadedPackage(packageName, packages)) {
+                    packageAdded = addPackage(packageName, packageAdded);
+                }
+                dotPos = 0;
+                do {
+                    dotPos = parentPackageName.lastIndexOf(DOT);
+                    if (dotPos > 0) {
+                        parentPackageName = parentPackageName.substring(0, dotPos);
+                        if (isLoadedPackage(parentPackageName, packages)) {
+                            packageAdded = addPackage(parentPackageName, packageAdded);
+                        }
+                    }
+                } while (dotPos > 0);
+
+                // handle package imports like 'from java import math'
+                for (String fromName : stringFromlist) {
+                    String fromPackageName = packageName + DOT + fromName;
+                    if (isLoadedPackage(fromPackageName, packages)) {
+                        packageAdded = addPackage(fromPackageName, packageAdded);
+                    }
+                }
+            }
+        }
+        return packageAdded;
+    }
+
+    /**
+     * Check if a java package is already known to the VM.
+     * <p>
+     * May return <code>false</code> even if the given package name is a valid java package !
+     *
+     * @param packageName
+     *
+     * @return <code>true</code> if the package with the given name is already loaded by the VM, <code>false</code>
+     * otherwise.
+     */
+    protected static boolean isLoadedPackage(String packageName) {
+        return isLoadedPackage(packageName, buildLoadedPackages());
+    }
+
+    /**
+     * Convert the fromlist into a java.lang.String based list.
+     * <p>
+     * Do some sanity checks: filter out '*' and empty tuples, as well as non tuples.
+     *
+     * @param fromlist
+     * @return a list containing java.lang.String entries
+     */
+    private static final List<String> getFromListAsStrings(PyObject fromlist) {
+        List<String> stringFromlist = new ArrayList<String>();
+
+        if (fromlist != null && fromlist != Py.EmptyTuple && fromlist instanceof PyTuple) {
+            Iterator iterator = ((PyTuple) fromlist).iterator();
+            while (iterator.hasNext()) {
+                Object obj = iterator.next();
+                if (obj instanceof PyString) {
+                    obj = ((PyString) obj).getString();
+                }
+                if (obj instanceof String) {
+                    String fromName = (String) obj;
+                    if (!"*".equals(fromName)) {
+                        stringFromlist.add(fromName);
+                    }
+                }
+            }
+        }
+        return stringFromlist;
+    }
+
+    /**
+     * Faster way to check if a java package is already known to the VM.
+     * <p>
+     * May return <code>false</code> even if the given package name is a valid java package !
+     *
+     * @param packageName
+     * @param packages A Map containing all packages actually known to the VM. Such a Map can be obtained using
+     * {@link JavaImportHelper.buildLoadedPackagesTree()}
+     *
+     * @return <code>true</code> if the package with the given name is already loaded by the VM, <code>false</code>
+     * otherwise.
+     */
+    private static boolean isLoadedPackage(String javaPackageName, Map<String, String> packages) {
+        boolean isLoaded = false;
+        if (javaPackageName != null) {
+            isLoaded = packages.containsKey(javaPackageName);
+        }
+        return isLoaded;
+    }
+
+    /**
+     * Build a <code>Map</code> of the currently known packages to the VM.
+     * <p>
+     * All parent packages appear as single entries like python modules, e.g. <code>java</code>,
+     * <code>java.lang</code>, <code>java.lang.reflect</code>,
+     */
+    private static Map<String, String> buildLoadedPackages() {
+        TreeMap<String, String> packageMap = new TreeMap<String, String>();
+        Package[] packages = Package.getPackages();
+        for (int i = 0; i < packages.length; i++) {
+            String packageName = packages[i].getName();
+            packageMap.put(packageName, "");
+            int dotPos = 0;
+            do {
+                dotPos = packageName.lastIndexOf(DOT);
+                if (dotPos > 0) {
+                    packageName = packageName.substring(0, dotPos);
+                    packageMap.put(packageName, "");
+                }
+            } while (dotPos > 0);
+        }
+        return packageMap;
+    }
+
+    /**
+     * @return <code>true</code> if the java class can be found by the current
+     *         Py classloader setup
+     */
+    private static boolean isJavaClass(String packageName, String className) {
+        return className != null && className.length() > 0
+                && Py.findClass(packageName + "." + className) != null;
+    }
+
+    /**
+     * Add a java package to sys.modules, if not already done
+     *
+     * @return <code>true</code> if something was really added, <code>false</code> otherwise
+     */
+    private static boolean addPackage(String packageName, boolean packageAdded) {
+        PyObject modules = Py.getSystemState().modules;
+        String internedPackageName = packageName.intern();
+        PyObject module = modules.__finditem__(internedPackageName);
+        // a previously failed import could have created a Py.None entry in sys.modules
+        if (module == null || module == Py.None) {
+            int dotPos = 0;
+            do {
+                PyJavaPackage p = PySystemState.add_package(packageName);
+                if(dotPos == 0) {
+                    modules.__setitem__(internedPackageName, p);
+                } else {
+                    module = modules.__finditem__(internedPackageName);
+                    if (module == null || module == Py.None) {
+                        modules.__setitem__(internedPackageName, p);
+                    }
+                }
+                dotPos = packageName.lastIndexOf(DOT);
+                if (dotPos > 0) {
+                    packageName = packageName.substring(0, dotPos);
+                    internedPackageName = packageName.intern();
+                }
+            } while(dotPos > 0);
+            // make sure not to turn off the packageAdded flag
+            packageAdded = true;
+        }
+        return packageAdded;
+    }
+
+}

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Thu Mar  7 18:10:44 2019
From: jython-checkins at python.org (jeff.allen)
Date: Thu, 07 Mar 2019 23:10:44 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Improve_comments_to_method?=
 =?utf-8?q?s_associated_with_import_processing=2E?=
Message-ID: <20190307231044.1.D44DF1E67290C393@mg.python.org>

https://hg.python.org/jython/rev/ce0f4236962b
changeset:   8220:ce0f4236962b
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Sat Feb 02 16:55:51 2019 +0000
summary:
  Improve comments to methods associated with import processing.

imp.java is a cornucopia of methods whose precise function it is difficult to
infer from names and parameters. The new comments are based on single-stepping
example imports, with a view to addressing #2654. Adjacent classes are also
affected. Code change is only for clarity, deferring suspect code.

files:
  src/org/python/core/BytecodeLoader.java   |  114 +-
  src/org/python/core/JavaImportHelper.java |   32 +-
  src/org/python/core/Py.java               |   80 +-
  src/org/python/core/PyModule.java         |   20 +-
  src/org/python/core/PyObject.java         |   12 +-
  src/org/python/core/PyRunnable.java       |    5 +-
  src/org/python/core/imp.java              |  625 +++++++--
  7 files changed, 629 insertions(+), 259 deletions(-)


diff --git a/src/org/python/core/BytecodeLoader.java b/src/org/python/core/BytecodeLoader.java
--- a/src/org/python/core/BytecodeLoader.java
+++ b/src/org/python/core/BytecodeLoader.java
@@ -1,31 +1,29 @@
 // Copyright (c) Corporation for National Research Initiatives
 package org.python.core;
 
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.List;
-import java.lang.reflect.Field;
+import org.objectweb.asm.ClassReader;
+
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.ObjectInputStream;
-
-import org.objectweb.asm.ClassReader;
-import org.python.util.Generic;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.LinkedList;
+import java.util.List;
 
 /**
- * Utility class for loading compiled python modules and java classes defined in python modules.
+ * Utility class for loading compiled Python modules and Java classes defined in Python modules.
  */
 public class BytecodeLoader {
 
     /**
-     * Turn the java byte code in data into a java class.
+     * Turn the Java class file data into a Java class.
      *
-     * @param name
-     *            the name of the class
-     * @param data
-     *            the java byte code.
-     * @param referents
-     *            superclasses and interfaces that the new class will reference.
+     * @param name fully-qualified binary name of the class
+     * @param data a class file as a byte array
+     * @param referents super-classes and interfaces that the new class will reference.
      */
     @SuppressWarnings("unchecked")
     public static Class<?> makeClass(String name, byte[] data, Class<?>... referents) {
@@ -33,24 +31,15 @@
         Loader loader = new Loader();
         for (Class<?> referent : referents) {
             try {
-                ClassLoader cur = referent.getClassLoader();
-                if (cur != null) {
-                    loader.addParent(cur);
-                }
-            } catch (SecurityException e) {
-            }
+                loader.addParent(referent.getClassLoader());
+            } catch (SecurityException e) {}
         }
         Class<?> c = loader.loadClassFromBytes(name, data);
         if (ContainsPyBytecode.class.isAssignableFrom(c)) {
             try {
                 fixPyBytecode((Class<? extends ContainsPyBytecode>) c);
-            } catch (IllegalAccessException e) {
-                throw new RuntimeException(e);
-            } catch (NoSuchFieldException e) {
-                throw new RuntimeException(e);
-            } catch (ClassNotFoundException e) {
-                throw new RuntimeException(e);
-            } catch (IOException e) {
+            } catch (IllegalAccessException | NoSuchFieldException | ClassNotFoundException
+                    | IOException e) {
                 throw new RuntimeException(e);
             }
         }
@@ -59,14 +48,11 @@
     }
 
     /**
-     * Turn the java byte code in data into a java class.
+     * Turn the Java class file data into a Java class.
      *
-     * @param name
-     *            the name of the class
-     * @param referents
-     *            superclasses and interfaces that the new class will reference.
-     * @param data
-     *            the java byte code.
+     * @param name the name of the class
+     * @param referents super-classes and interfaces that the new class will reference.
+     * @param data a class file as a byte array
      */
     public static Class<?> makeClass(String name, List<Class<?>> referents, byte[] data) {
         if (referents != null) {
@@ -92,7 +78,7 @@
      * decoder treats characters outside the set of 64 necessary to encode data as errors, including
      * the pad "=". As a result, the length of the argument exactly determines the size of array
      * returned.
-     * 
+     *
      * @param src to decode
      * @return a new byte array
      * @throws IllegalArgumentException if src has an invalid character or impossible length.
@@ -139,6 +125,7 @@
 
     /**
      * Helper for {@link #base64decode(String)}, converting one character.
+     *
      * @param c to convert
      * @return value 0..63
      * @throws IllegalArgumentException if not a base64 character
@@ -146,7 +133,7 @@
     private static int base64CharToBits(char c) throws IllegalArgumentException {
         if (c >= 'a') {
             if (c <= 'z') {
-                return c - ('a' - 26);
+                return c - 71; // c - 'a' + 26
             }
         } else if (c >= 'A') {
             if (c <= 'Z') {
@@ -154,7 +141,7 @@
             }
         } else if (c >= '0') {
             if (c <= '9') {
-                return c + (52 - '0');
+                return c + 4; // c - '0' + 52
             }
         } else if (c == '+') {
             return 62;
@@ -191,14 +178,14 @@
      * special treatment after class-loading.
      */
     public static void fixPyBytecode(Class<? extends ContainsPyBytecode> c)
-            throws IllegalAccessException, NoSuchFieldException, java.io.IOException, ClassNotFoundException
-    {
+            throws IllegalAccessException, NoSuchFieldException, java.io.IOException,
+            ClassNotFoundException {
         Field[] fields = c.getDeclaredFields();
         for (Field fld: fields) {
             String fldName = fld.getName();
             if (fldName.startsWith("___")) {
                 fldName = fldName.substring(3);
-                
+
                 String[] splt = fldName.split("_");
                 if (splt[0].equals("0")) {
                     fldName = fldName.substring(2);
@@ -227,13 +214,16 @@
                                         if (Integer.parseInt(splt2[1]) == pos) {
                                             blt.append((String) fldPart.get(null));
                                             pos += 1;
-                                            if (pos == len) break;
+                                            if (pos == len) {
+                                                break;
+                                            }
                                         }
                                     }
                                 }
                                 if (pos0 == pos) {
-                                    throw new RuntimeException("Invalid PyBytecode splitting in "+c.getName()+
-                                            ":\nSplit-index "+pos+" wasn't found.");
+                                    throw new RuntimeException(
+                                            "Invalid PyBytecode splitting in " + c.getName()
+                                                    + ":\nSplit-index " + pos + " wasn't found.");
                                 }
                             }
                             codeField.set(null, parseSerializedCode(blt.toString()));
@@ -245,20 +235,22 @@
     }
 
     /**
-     * Turn the java byte code for a compiled python module into a java class.
+     * Turn the Java class file data for a compiled Python module into a {@code PyCode} object, by
+     * constructing an instance of the named class and calling the instance's
+     * {@link PyRunnable#getMain()}.
      *
-     * @param name
-     *            the name of the class
-     * @param data
-     *            the java byte code.
+     * @param name fully-qualified binary name of the class
+     * @param data a class file as a byte array
+     * @param filename to provide to the constructor of the named class
+     * @return the {@code PyCode} object produced by the named class' {@code getMain}
      */
     public static PyCode makeCode(String name, byte[] data, String filename) {
         try {
             Class<?> c = makeClass(name, data);
-            Object o = c.getConstructor(new Class[] {String.class})
-                    .newInstance(new Object[] {filename});
-
-            PyCode result = ((PyRunnable)o).getMain();
+            // A compiled module has a constructor taking a String filename argument.
+            Constructor<?> cons = c.getConstructor(new Class<?>[] {String.class});
+            Object instance = cons.newInstance(new Object[] {filename});
+            PyCode result = ((PyRunnable) instance).getMain();
             return result;
         } catch (Exception e) {
             throw Py.JavaError(e);
@@ -267,16 +259,17 @@
 
     public static class Loader extends URLClassLoader {
 
-        private List<ClassLoader> parents = Generic.list();
+        private LinkedList<ClassLoader> parents = new LinkedList<>();
 
         public Loader() {
             super(new URL[0]);
             parents.add(imp.getSyspathJavaLoader());
         }
 
+        /** Add given loader at the front of the list of the parent list (if not {@code null}). */
         public void addParent(ClassLoader referent) {
-            if (!parents.contains(referent)) {
-                parents.add(0, referent);
+            if (referent != null && !parents.contains(referent)) {
+                parents.addFirst(referent);
             }
         }
 
@@ -295,6 +288,15 @@
             throw new ClassNotFoundException(name);
         }
 
+        /**
+         * Define the named class using the class file data provided, and resolve it. (See JVM
+         * specification.) For class names ending "$py", this method may adjust that name to that
+         * found in the class file itself.
+         *
+         * @param name fully-qualified binary name of the class
+         * @param data a class file as a byte array
+         * @return the defined and resolved class
+         */
         public Class<?> loadClassFromBytes(String name, byte[] data) {
             if (name.endsWith("$py")) {
                 try {
diff --git a/src/org/python/core/JavaImportHelper.java b/src/org/python/core/JavaImportHelper.java
--- a/src/org/python/core/JavaImportHelper.java
+++ b/src/org/python/core/JavaImportHelper.java
@@ -16,17 +16,18 @@
     /**
      * Try to add the java package.
      * <p>
-     * This is handy in cases where the package scan cannot run, or when the initial classpath does not contain all .jar
-     * files (such as in J2EE containers).
+     * This is handy in cases where the package scan cannot run, or when the initial classpath does
+     * not contain all .jar files (such as in J2EE containers).
      * <p>
-     * There is some self-healing in the sense that a correct, explicit import of a java class will succeed even if
-     * sys.modules already contains a Py.None entry for the corresponding java package.
+     * There is some self-healing in the sense that a correct, explicit import of a java class will
+     * succeed even if sys.modules already contains a Py.None entry for the corresponding java
+     * package.
      *
      * @param packageName The dotted name of the java package
      * @param fromlist A tuple with the from names to import. Can be null or empty.
      *
-     * @return <code>true</code> if a java package was doubtlessly identified and added, <code>false</code>
-     * otherwise.
+     * @return <code>true</code> if a java package was doubtlessly identified and added,
+     *         <code>false</code> otherwise.
      */
     protected static boolean tryAddPackage(final String packageName, PyObject fromlist) {
         // make sure we do not turn off the added flag, once it is set
@@ -94,8 +95,8 @@
      *
      * @param packageName
      *
-     * @return <code>true</code> if the package with the given name is already loaded by the VM, <code>false</code>
-     * otherwise.
+     * @return <code>true</code> if the package with the given name is already loaded by the VM,
+     *         <code>false</code> otherwise.
      */
     protected static boolean isLoadedPackage(String packageName) {
         return isLoadedPackage(packageName, buildLoadedPackages());
@@ -136,11 +137,11 @@
      * May return <code>false</code> even if the given package name is a valid java package !
      *
      * @param packageName
-     * @param packages A Map containing all packages actually known to the VM. Such a Map can be obtained using
-     * {@link JavaImportHelper.buildLoadedPackagesTree()}
+     * @param packages A Map containing all packages actually known to the VM. Such a Map can be
+     *            obtained using {@link JavaImportHelper.buildLoadedPackagesTree()}
      *
-     * @return <code>true</code> if the package with the given name is already loaded by the VM, <code>false</code>
-     * otherwise.
+     * @return <code>true</code> if the package with the given name is already loaded by the VM,
+     *         <code>false</code> otherwise.
      */
     private static boolean isLoadedPackage(String javaPackageName, Map<String, String> packages) {
         boolean isLoaded = false;
@@ -175,8 +176,9 @@
     }
 
     /**
-     * @return <code>true</code> if the java class can be found by the current
-     *         Py classloader setup
+     * Try to load <i>packageName.className</i> and return {@code true} if successful.
+     *
+     * @return <code>true</code> if the java class can be found by the current Py classloader setup
      */
     private static boolean isJavaClass(String packageName, String className) {
         return className != null && className.length() > 0
@@ -184,7 +186,7 @@
     }
 
     /**
-     * Add a java package to sys.modules, if not already done
+     * Add a java package to sys.modules, if not already done.
      *
      * @return <code>true</code> if something was really added, <code>false</code> otherwise
      */
diff --git a/src/org/python/core/Py.java b/src/org/python/core/Py.java
--- a/src/org/python/core/Py.java
+++ b/src/org/python/core/Py.java
@@ -1063,99 +1063,97 @@
     private static boolean syspathJavaLoaderRestricted = false;
 
     /**
-     * Common code for findClass and findClassEx
-     * @param name Name of the Java class to load and initialize
-     * @param reason Reason for loading it, used for debugging. No debug output
-     *               is generated if it is null
+     * Common code for {@link #findClass(String)} and {@link #findClassEx(String, String)}.
+     *
+     * @param name of the Java class to load and initialise
+     * @param reason to be given in debug output (or {@code null} to suppress debug output.
      * @return the loaded class
      * @throws ClassNotFoundException if the class wasn't found by the class loader
      */
-    private static Class<?> findClassInternal(String name, String reason) throws ClassNotFoundException {
+    private static Class<?> findClassInternal(String name, String reason)
+            throws ClassNotFoundException {
+
         ClassLoader classLoader = Py.getSystemState().getClassLoader();
         if (classLoader != null) {
             if (reason != null) {
-                writeDebug("import", "trying " + name + " as " + reason +
-                          " in sys.classLoader");
+                writeDebug("import", "trying " + name + " as " + reason + " in sys.classLoader");
             }
             return loadAndInitClass(name, classLoader);
         }
+
         if (!syspathJavaLoaderRestricted) {
             try {
                 classLoader = imp.getSyspathJavaLoader();
                 if (classLoader != null && reason != null) {
-                    writeDebug("import", "trying " + name + " as " + reason +
-                            " in SysPathJavaLoader");
+                    writeDebug("import",
+                            "trying " + name + " as " + reason + " in SysPathJavaLoader");
                 }
             } catch (SecurityException e) {
                 syspathJavaLoaderRestricted = true;
             }
         }
+
         if (syspathJavaLoaderRestricted) {
             classLoader = imp.getParentClassLoader();
             if (classLoader != null && reason != null) {
-                writeDebug("import", "trying " + name + " as " + reason +
-                        " in Jython's parent class loader");
+                writeDebug("import",
+                        "trying " + name + " as " + reason + " in Jython's parent class loader");
             }
         }
+
         if (classLoader != null) {
             try {
                 return loadAndInitClass(name, classLoader);
             } catch (ClassNotFoundException cnfe) {
                 // let the default classloader try
-                // XXX: by trying another classloader that may not be on a
-                //      parent/child relationship with the Jython's parent
-                //      classsloader we are risking some nasty class loading
-                //      problems (such as having two incompatible copies for
-                //      the same class that is itself a dependency of two
-                //      classes loaded from these two different class loaders)
+                /*
+                 * XXX: by trying another classloader that may not be on a parent/child relationship
+                 * with the Jython's parent classsloader we are risking some nasty class loading
+                 * problems (such as having two incompatible copies for the same class that is
+                 * itself a dependency of two classes loaded from these two different class
+                 * loaders).
+                 */
             }
         }
+
         if (reason != null) {
-            writeDebug("import", "trying " + name + " as " + reason +
-                       " in context class loader, for backwards compatibility");
+            writeDebug("import", "trying " + name + " as " + reason
+                    + " in context class loader, for backwards compatibility");
         }
+
         return loadAndInitClass(name, Thread.currentThread().getContextClassLoader());
     }
 
     /**
-     * Tries to find a Java class.
-     * @param name Name of the Java class.
-     * @return The class, or null if it wasn't found
+     * Find and load a Java class by name.
+     *
+     * @param name of the Java class.
+     * @return the class, or {@code null} if it wasn't found or something went wrong
      */
     public static Class<?> findClass(String name) {
         try {
             return findClassInternal(name, null);
-        } catch (ClassNotFoundException e) {
-            //             e.printStackTrace();
-            return null;
-        } catch (IllegalArgumentException e) {
-            //             e.printStackTrace();
-            return null;
-        } catch (NoClassDefFoundError e) {
-            //             e.printStackTrace();
+        } catch (ClassNotFoundException | IllegalArgumentException | NoClassDefFoundError e) {
+            // e.printStackTrace();
             return null;
         }
     }
 
     /**
-     * Tries to find a Java class.
+     * Find and load a Java class by name.
      *
-     * Unless {@link #findClass(String)}, it raises a JavaError
-     * if the class was found but there were problems loading it.
      * @param name Name of the Java class.
-     * @param reason Reason for finding the class. Used for debugging messages.
-     * @return The class, or null if it wasn't found
-     * @throws JavaError wrapping LinkageErrors/IllegalArgumentExceptions
-     * occurred when the class is found but can't be loaded.
+     * @param reason for finding the class. Used in debugging messages.
+     * @return the class, or {@code null} if it simply wasn't found
+     * @throws PyException (JavaError) wrapping errors occurring when the class is found but cannot
+     *             be loaded.
      */
-    public static Class<?> findClassEx(String name, String reason) {
+    public static Class<?> findClassEx(String name, String reason) throws PyException {
         try {
             return findClassInternal(name, reason);
         } catch (ClassNotFoundException e) {
             return null;
-        } catch (IllegalArgumentException e) {
-            throw JavaError(e);
-        } catch (LinkageError e) {
+        } catch (IllegalArgumentException | LinkageError e) {
             throw JavaError(e);
         }
     }
diff --git a/src/org/python/core/PyModule.java b/src/org/python/core/PyModule.java
--- a/src/org/python/core/PyModule.java
+++ b/src/org/python/core/PyModule.java
@@ -14,7 +14,7 @@
  */
 @ExposedType(name = "module")
 public class PyModule extends PyObject implements Traverseproc {
-    private final PyObject moduleDoc = new PyString(
+    private final PyObject moduleDoc = new PyString( //FIXME: not used (and not static)
         "module(name[, doc])\n" +
         "\n" +
         "Create a module object.\n" +
@@ -88,8 +88,16 @@
         throw Py.TypeError("readonly attribute");
     }
 
+    /**
+     * {@inheritDoc}
+     * <p>
+     * Overridden in {@code PyModule} to search for the named attribute as a
+     * module in {@code sys.modules} (using the key {@code ".".join(self.__name__, name)}) and on the
+     * {@code self.__path__}.
+     */
     @Override
     protected PyObject impAttr(String name) {
+        // Get hold of the module dictionary and __name__, and __path__ if it has one.
         if (__dict__ == null || name.length() == 0) {
             return null;
         }
@@ -102,10 +110,12 @@
             return null;
         }
 
+        // Maybe the module we're looking for is in sys.modules
         String fullName = (pyName.__str__().toString() + '.' + name).intern();
         PyObject modules = Py.getSystemState().modules;
         PyObject attr = modules.__finditem__(fullName);
 
+        // If not, look along the module's __path__
         if (path instanceof PyList) {
             if (attr == null) {
                 attr = imp.find_module(name, fullName, (PyList)path);
@@ -115,6 +125,7 @@
         }
 
         if (attr == null) {
+            // Still looking: maybe it's a Java package?
             attr = PySystemState.packageManager.lookupName(fullName);
         }
 
@@ -131,6 +142,13 @@
         return null;
     }
 
+    /**
+     * {@inheritDoc}
+     * <p>
+     * Overridden in {@code PyModule} so that if the base-class {@code __findattr_ex__} is
+     * unsuccessful, it will to search for the named attribute as a module via
+     * {@link #impAttr(String)}.
+     */
     @Override
     public PyObject __findattr_ex__(String name) {
         PyObject attr = super.__findattr_ex__(name);
diff --git a/src/org/python/core/PyObject.java b/src/org/python/core/PyObject.java
--- a/src/org/python/core/PyObject.java
+++ b/src/org/python/core/PyObject.java
@@ -897,7 +897,7 @@
      *
      * <b>Warning: name must be an interned string!</b>
      *
-     * @param name the name to lookup in this namespace <b> must be an interned string </b>.
+     * @param name the name to lookup in this namespace <b>must be an interned string</b>.
      * @return the value corresponding to name or null if name is not found
      **/
     public final PyObject __findattr__(String name) {
@@ -1027,7 +1027,15 @@
         object___delattr__(name);
     }
 
-    // Used by import logic.
+    /**
+     * This is a variant of {@link #__findattr__(String)} used by the module import logic to find a
+     * sub-module amongst the attributes of an object representing a package. The default behaviour
+     * is to delegate to {@code __findattr__}, but in particular cases it becomes a hook for
+     * specialised search behaviour.
+     *
+     * @param name the name to lookup in this namespace <b>must be an interned string</b>.
+     * @return the value corresponding to name or null if name is not found
+     */
     protected PyObject impAttr(String name) {
         return __findattr__(name);
     }
diff --git a/src/org/python/core/PyRunnable.java b/src/org/python/core/PyRunnable.java
--- a/src/org/python/core/PyRunnable.java
+++ b/src/org/python/core/PyRunnable.java
@@ -5,10 +5,7 @@
  * Interface implemented by compiled modules which allow access to
  * to the module code object.
  */
-
 public interface PyRunnable {
-    /**
-     * Return the modules code object.
-     */
+    /** Return the module's code object. */
     abstract public PyCode getMain();
 }
diff --git a/src/org/python/core/imp.java b/src/org/python/core/imp.java
--- a/src/org/python/core/imp.java
+++ b/src/org/python/core/imp.java
@@ -1,6 +1,10 @@
 // Copyright (c) Corporation for National Research Initiatives
 package org.python.core;
 
+import org.python.compiler.Module;
+import org.python.core.util.FileUtil;
+import org.python.core.util.PlatformUtil;
+
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
@@ -10,10 +14,6 @@
 import java.util.Map;
 import java.util.concurrent.locks.ReentrantLock;
 
-import org.python.compiler.Module;
-import org.python.core.util.FileUtil;
-import org.python.core.util.PlatformUtil;
-
 /**
  * Utility functions for "import" support.
  *
@@ -38,6 +38,12 @@
 
     private static final boolean IS_OSX = PySystemState.getNativePlatform().equals("darwin");
 
+    /**
+     * A bundle of a file name, the file's content and a last modified time, with no behaviour. As
+     * used here, the file is a class file and the last modified time is that of the matching
+     * source, while the filename is taken from the annotation in the class file. See
+     * {@link imp#readCodeData(String, InputStream, boolean, long)}.
+     */
     public static class CodeData {
 
         private final byte[] bytes;
@@ -63,8 +69,18 @@
         }
     }
 
-    public static enum CodeImport {
-        source, compiled_only;
+    /**
+     * A two-way selector given to
+     * {@link imp#createFromPyClass(String, InputStream, boolean, String, String, long, CodeImport)}
+     * that tells it whether the source file name to give the module-class constructor, and which
+     * ends up in {@code co_filename} attribute of the module's {@code PyCode}, should be from the
+     * caller or the compiled file.
+     */
+    static enum CodeImport {
+        /** Take the filename from the {@code sourceName} argument */
+        source,
+        /** Take filename from the compiler annotation */
+        compiled_only;
     }
 
     /** A non-empty fromlist for __import__'ing sub-modules. */
@@ -76,11 +92,11 @@
 
     /**
      * Selects the parent class loader for Jython, to be used for dynamically loaded classes and
-     * resources. Chooses between the current and context classloader based on the following
+     * resources. Chooses between the current and context class loader based on the following
      * criteria:
      *
      * <ul>
-     * <li>If both are the same classloader, return that classloader.
+     * <li>If both are the same class loader, return that class loader.
      * <li>If either is null, then the non-null one is selected.
      * <li>If both are not null, and a parent/child relationship can be determined, then the child
      * is selected.
@@ -89,62 +105,61 @@
      * Jython classes)
      * </ul>
      *
-     * @return the parent class loader for Jython or null if both the current and context
-     *         classloaders are null.
+     * @return the parent class loader for Jython or null if both the current and context class
+     *         loaders are null.
      */
     public static ClassLoader getParentClassLoader() {
         ClassLoader current = imp.class.getClassLoader();
         ClassLoader context = Thread.currentThread().getContextClassLoader();
-        if (context == current) {
-            return current;
-        }
-        if (context == null) {
+        if (context == current || context == null) {
             return current;
-        }
-        if (current == null) {
+        } else if (current == null) {
             return context;
-        }
-        if (isParentClassLoader(context, current)) {
+        } else if (isAncestor(context, current)) {
+            return current;
+        } else if (isAncestor(current, context)) {
+            return context;
+        } else {
             return current;
         }
-        if (isParentClassLoader(current, context)) {
-            return context;
-        }
-        return current;
     }
 
-    private static boolean isParentClassLoader(ClassLoader suspectedParent, ClassLoader child) {
+    /** True iff a {@code possibleAncestor} is the ancestor of the {@code subject}. */
+    private static boolean isAncestor(ClassLoader possibleAncestor, ClassLoader subject) {
         try {
-            ClassLoader parent = child.getParent();
-            if (suspectedParent == parent) {
+            ClassLoader parent = subject.getParent();
+            if (possibleAncestor == parent) {
                 return true;
+            } else if (parent == null || parent == subject) {
+                // The subject is the boot class loader
+                return false;
+            } else {
+                return isAncestor(possibleAncestor, parent);
             }
-            if (parent == null || parent == child) {
-                // We reached the boot class loader
-                return false;
-            }
-            return isParentClassLoader(suspectedParent, parent);
-
         } catch (SecurityException e) {
             return false;
         }
     }
 
-    private imp() {}
+    private imp() {} // Prevent instantiation.
 
     /**
      * If the given name is found in sys.modules, the entry from there is returned. Otherwise a new
-     * PyModule is created for the name and added to sys.modules
+     * {@link PyModule} is created for the name and added to {@code sys.modules}. Creating the
+     * module does not execute the body of the module to initialise its attributes.
+     *
+     * @param name fully-qualified name of the module
+     * @return created {@code PyModule}
      */
     public static PyModule addModule(String name) {
         name = name.intern();
         PyObject modules = Py.getSystemState().modules;
-        PyModule module = (PyModule)modules.__finditem__(name);
+        PyModule module = (PyModule) modules.__finditem__(name);
         if (module != null) {
             return module;
         }
         module = new PyModule(name, null);
-        PyModule __builtin__ = (PyModule)modules.__finditem__("__builtin__");
+        PyModule __builtin__ = (PyModule) modules.__finditem__("__builtin__");
         PyObject __dict__ = module.__getattr__("__dict__");
         __dict__.__setitem__("__builtins__", __builtin__.__getattr__("__dict__"));
         __dict__.__setitem__("__package__", Py.None);
@@ -153,7 +168,7 @@
     }
 
     /**
-     * Remove name form sys.modules if it's there.
+     * Remove name from sys.modules if present.
      *
      * @param name the module name
      */
@@ -172,6 +187,12 @@
         }
     }
 
+    /**
+     * Read a stream as a new byte array and close the stream.
+     *
+     * @param fp to read
+     * @return bytes read
+     */
     private static byte[] readBytes(InputStream fp) {
         try {
             return FileUtil.readBytes(fp);
@@ -186,6 +207,7 @@
         }
     }
 
+    /** Open a file, raising a {@code PyException} on error. */
     private static InputStream makeStream(File file) {
         try {
             return new FileInputStream(file);
@@ -194,23 +216,69 @@
         }
     }
 
+    /**
+     * As {@link #createFromPyClass(String, InputStream, boolean, String, String, long, CodeImport)}
+     * but always constructs the named class using {@code sourceName} as argument and makes no check
+     * on the last-modified time.
+     *
+     * @param name module name on which to base the class name as {@code name + "$py"}
+     * @param fp stream from which to read class file (closed when read)
+     * @param testing if {@code true}, failures are signalled by a {@code null} not an exception
+     * @param sourceName used for identification in messages and the constructor of the named class.
+     * @param compiledName used for identification in messages and {@code __file__}.
+     * @return the module or {@code null} on failure (if {@code testing}).
+     * @throws PyException (ImportError) on API mismatch or i/o error.
+     */
     static PyObject createFromPyClass(String name, InputStream fp, boolean testing,
             String sourceName, String compiledName) {
         return createFromPyClass(name, fp, testing, sourceName, compiledName, NO_MTIME);
-
     }
 
+    /**
+     * As {@link #createFromPyClass(String, InputStream, boolean, String, String, long, CodeImport)}
+     * but always constructs the named class using {@code sourceName} as argument.
+     *
+     * @param name module name on which to base the class name as {@code name + "$py"}
+     * @param fp stream from which to read class file (closed when read)
+     * @param testing if {@code true}, failures are signalled by a {@code null} not an exception
+     * @param sourceName used for identification in messages and the constructor of the named class.
+     * @param compiledName used for identification in messages and {@code __file__}.
+     * @param sourceLastModified time expected to match {@code MTime} annotation in the class file
+     * @return the module or {@code null} on failure (if {@code testing}).
+     * @throws PyException (ImportError) on API or last-modified time mismatch or i/o error.
+     */
     static PyObject createFromPyClass(String name, InputStream fp, boolean testing,
-            String sourceName, String compiledName, long mtime) {
-        return createFromPyClass(name, fp, testing, sourceName, compiledName, mtime,
+            String sourceName, String compiledName, long sourceLastModified) {
+        return createFromPyClass(name, fp, testing, sourceName, compiledName, sourceLastModified,
                 CodeImport.source);
     }
 
+    /**
+     * Create a Python module from its compiled form, reading the class file from the open input
+     * stream passed in (which is closed once read). The method may be used in a "testing" mode in
+     * which the module is imported (if possible), but error conditions return {@code null}, or in a
+     * non-testing mode where they throw. The caller may choose whether the source file name to give
+     * the module-class constructor, and which ends up in {@code co_filename} attribute of the
+     * module's {@code PyCode}, should be {@code sourceName} or the compiled file (See
+     * {@link CodeImport}.)
+     *
+     * @param name module name on which to base the class name as {@code name + "$py"}
+     * @param fp stream from which to read class file (closed when read)
+     * @param testing if {@code true}, failures are signalled by a {@code null} not an exception
+     * @param sourceName used for identification in messages.
+     * @param compiledName used for identification in messages and {@code __file__}.
+     * @param sourceLastModified time expected to match {@code MTime} annotation in the class file
+     * @param source choose what to use as the file name when initialising the class
+     * @return the module or {@code null} on failure (if {@code testing}).
+     * @throws PyException (ImportError) on API or last-modified time mismatch or i/o error.
+     */
     static PyObject createFromPyClass(String name, InputStream fp, boolean testing,
-            String sourceName, String compiledName, long mtime, CodeImport source) {
+            String sourceName, String compiledName, long sourceLastModified, CodeImport source) {
+
+        // Get the contents of a compiled ($py.class) file and some meta-data
         CodeData data = null;
         try {
-            data = readCodeData(compiledName, fp, testing, mtime);
+            data = readCodeData(compiledName, fp, testing, sourceLastModified);
         } catch (IOException ioe) {
             if (!testing) {
                 throw Py.ImportError(ioe.getMessage() + "[name=" + name + ", source=" + sourceName
@@ -220,10 +288,13 @@
         if (testing && data == null) {
             return null;
         }
+
+        // Instantiate the class and have it produce its PyCode object.
         PyCode code;
         try {
-            code = BytecodeLoader.makeCode(name + "$py", data.getBytes(), //
-                    source == CodeImport.compiled_only ? data.getFilename() : sourceName);
+            // Choose which file name to provide to the module-class constructor
+            String display = source == CodeImport.compiled_only ? data.getFilename() : sourceName;
+            code = BytecodeLoader.makeCode(name + "$py", data.getBytes(), display);
         } catch (Throwable t) {
             if (testing) {
                 return null;
@@ -232,18 +303,42 @@
             }
         }
 
+        // Execute the PyCode object (run the module body) to populate the module __dict__
         Py.writeComment(IMPORT_LOG,
                 String.format("import %s # precompiled from %s", name, compiledName));
         return createFromCode(name, code, compiledName);
     }
 
+    /**
+     * As {@link #readCodeData(String, InputStream, boolean, long)} but do not check last-modified
+     * time and return only the class file bytes as an array.
+     *
+     * @param name of source file (used for identification in error/log messages)
+     * @param fp stream from which to read class file (closed when read)
+     * @param testing if {@code true}, failures are signalled by a {@code null} not an exception
+     * @return the class file bytes as an array or {@code null} on failure (if {@code testing}).
+     * @throws PyException (ImportError) on API or last-modified time mismatch
+     * @throws IOException from read failures
+     */
     public static byte[] readCode(String name, InputStream fp, boolean testing) throws IOException {
         return readCode(name, fp, testing, NO_MTIME);
     }
 
-    public static byte[] readCode(String name, InputStream fp, boolean testing, long mtime)
-            throws IOException {
-        CodeData data = readCodeData(name, fp, testing, mtime);
+    /**
+     * As {@link #readCodeData(String, InputStream, boolean, long)} but return only the class file
+     * bytes as an array.
+     *
+     * @param name of source file (used for identification in error/log messages)
+     * @param fp stream from which to read class file (closed when read)
+     * @param testing if {@code true}, failures are signalled by a {@code null} not an exception
+     * @param sourceLastModified time expected to match {@code MTime} annotation in the class file
+     * @return the class file bytes as an array or {@code null} on failure (if {@code testing}).
+     * @throws PyException (ImportError) on API or last-modified time mismatch
+     * @throws IOException from read failures
+     */
+    public static byte[] readCode(String name, InputStream fp, boolean testing,
+            long sourceLastModified) throws IOException {
+        CodeData data = readCodeData(name, fp, testing, sourceLastModified);
         if (data == null) {
             return null;
         } else {
@@ -251,17 +346,45 @@
         }
     }
 
+    /**
+     * As {@link #readCodeData(String, InputStream, boolean, long)} but do not check last-modified
+     * time.
+     *
+     * @param name of source file (used for identification in error/log messages)
+     * @param fp stream from which to read class file (closed when read)
+     * @param testing if {@code true}, failures are signalled by a {@code null} not an exception
+     * @return the {@code CodeData} bundle or {@code null} on failure (if {@code testing}).
+     * @throws PyException (ImportError) on API mismatch
+     * @throws IOException from read failures
+     */
     public static CodeData readCodeData(String name, InputStream fp, boolean testing)
             throws IOException {
         return readCodeData(name, fp, testing, NO_MTIME);
     }
 
-    public static CodeData readCodeData(String name, InputStream fp, boolean testing, long mtime)
-            throws IOException {
-        byte[] data = readBytes(fp);
-        int api;
-        AnnotationReader ar = new AnnotationReader(data);
-        api = ar.getVersion();
+    /**
+     * Create a {@link CodeData} object bundling the contents of a class file (given as a stream),
+     * source-last-modified time supplied, and the name of the file taken from annotations on the
+     * class. On the way, the method checks the API version annotation matches the current process,
+     * and that the {@code org.python.compiler.MTime} annotation matches the source-last-modified
+     * time passed in.
+     *
+     * @param name of source file (used for identification in error/log messages)
+     * @param fp stream from which to read class file (closed when read)
+     * @param testing if {@code true}, failures are signalled by a {@code null} not an exception
+     * @param sourceLastModified time expected to match {@code MTime} annotation in the class file
+     * @return the {@code CodeData} bundle or {@code null} on failure (if {@code testing}).
+     * @throws PyException (ImportError) on API or last-modified time mismatch
+     * @throws IOException from read failures
+     */
+    public static CodeData readCodeData(String name, InputStream fp, boolean testing,
+            long sourceLastModified) throws IOException, PyException {
+
+        byte[] classFileData = readBytes(fp);
+        AnnotationReader ar = new AnnotationReader(classFileData);
+
+        // Check API version fossilised in the class file against that expected
+        int api = ar.getVersion();
         if (api != APIVersion) {
             if (testing) {
                 return null;
@@ -270,13 +393,17 @@
                 throw Py.ImportError(String.format(fmt, api, APIVersion, name));
             }
         }
-        if (testing && mtime != NO_MTIME) {
-            long time = ar.getMTime();
-            if (mtime != time) {
+
+        // Check source-last-modified time fossilised in the class file against that expected
+        if (testing && sourceLastModified != NO_MTIME) {
+            long mtime = ar.getMTime();
+            if (sourceLastModified != mtime) {
                 return null;
             }
         }
-        return new CodeData(data, mtime, ar.getFilename());
+
+        // All well: make the bundle.
+        return new CodeData(classFileData, sourceLastModified, ar.getFilename());
     }
 
     public static byte[] compileSource(String name, File file) {
@@ -383,6 +510,16 @@
         return createFromSource(name, fp, filename, outFilename, NO_MTIME);
     }
 
+    /**
+     * Compile Jython source (as an {@code InputStream}) to a module.
+     *
+     * @param name of the module to create (class will be name$py)
+     * @param fp opened on the (Jython) source to compile (will be closed)
+     * @param filename of the source backing {@code fp} (to embed in class as data)
+     * @param outFilename in which to write the compiled class
+     * @param mtime last modified time of the file backing {@code fp}
+     * @return created module
+     */
     public static PyObject createFromSource(String name, InputStream fp, String filename,
             String outFilename, long mtime) {
         byte[] bytes = compileSource(name, fp, filename, mtime);
@@ -405,11 +542,18 @@
     }
 
     /**
-     * Returns a module with the given name whose contents are the results of running c. Sets
-     * __file__ on the module to be moduleLocation unless moduleLocation is null. If c comes from a
-     * local .py file or compiled $py.class class moduleLocation should be the result of running new
-     * File(moduleLocation).getAbsolutePath(). If c comes from a remote file or is a jar
-     * moduleLocation should be the full uri for c.
+     * Return a Python module with the given {@code name} whose attributes are the result of running
+     * {@code PyCode c}. If {@code moduleLocation != null} it is used to set {@code __file__ }.
+     * <p>
+     * In normal circumstances, if {@code c} comes from a local {@code .py} file or compiled
+     * {@code $py.class} class the caller should should set {@code moduleLocation} to something like
+     * {@code new File(moduleLocation).getAbsolutePath()}. If {@code c} comes from a remote file or
+     * is a JAR, {@code moduleLocation} should be the full URI for that source or class.
+     *
+     * @param name fully-qualified name of the module
+     * @param c code supplying the module
+     * @param moduleLocation to become {@code __file__} if not {@code null}
+     * @return the module object
      */
     public static PyObject createFromCode(String name, PyCode c, String moduleLocation) {
         checkName(name);
@@ -417,7 +561,7 @@
 
         PyBaseCode code = null;
         if (c instanceof PyBaseCode) {
-            code = (PyBaseCode)c;
+            code = (PyBaseCode) c;
         }
 
         if (moduleLocation != null) {
@@ -466,18 +610,30 @@
         return getPathImporter(sys.path_importer_cache, sys.path_hooks, p);
     }
 
+    /**
+     * Return an importer object for an element of {@code sys.path} or of a package's
+     * {@code __path__}, possibly by fetching it from the {@code cache}. If it wasn?t yet cached,
+     * traverse {@code hooks} until a hook is found that can handle the path item. Return
+     * {@link Py#None} if no hook could do so. This tells our caller it should fall back to the
+     * built-in import mechanism. Cache the result in {@code cache}. Return a new reference to the
+     * importer object.
+     * <p>
+     * This is the "path hooks" mechanism first described in PEP 302
+     *
+     * @param cache normally {@code sys.path_importer_cache}
+     * @param hooks normally (@code sys.path_hooks}
+     * @param p an element of {@code sys.path} or of a package's {@code __path__}
+     * @return the importer object for the path element or {@code Py.None} for "fall-back".
+     */
     static PyObject getPathImporter(PyObject cache, PyList hooks, PyObject p) {
 
-        // attempt to get an importer for the path
-        // use null as default value since Py.None is
-        // a valid value in the cache for the default
-        // importer
+        // Is it in the cache?
         PyObject importer = cache.__finditem__(p);
         if (importer != null) {
             return importer;
         }
 
-        // nothing in the cache, so check all hooks
+        // Nothing in the cache, so check all hooks.
         PyObject iter = hooks.__iter__();
         for (PyObject hook; (hook = iter.__iternext__()) != null;) {
             try {
@@ -491,6 +647,7 @@
         }
 
         if (importer == null) {
+            // No hook claims to handle the location p, so add an imp.NullImporter
             try {
                 importer = new PyNullImporter(p);
             } catch (PyException e) {
@@ -501,19 +658,32 @@
         }
 
         if (importer != null) {
+            // We found an importer. Cache it for next time.
             cache.__setitem__(p, importer);
         } else {
+            // Caller will fall-back to built-in mechanisms.
             importer = Py.None;
         }
 
         return importer;
     }
 
+    /**
+     * Try to load a module from {@code sys.meta_path}, as a built-in module, or from either the the
+     * {@code __path__} of the enclosing package or {@code sys.path} if the module is being sought
+     * at the top level.
+     *
+     * @param name simple name of the module.
+     * @param moduleName fully-qualified (dotted) name of the module (ending in {@code name}.
+     * @param path {@code __path__} of the enclosing package (or {@code null} if top level).
+     * @return the module if we can load it (or {@code null} if we can't).
+     */
     static PyObject find_module(String name, String moduleName, PyList path) {
         PyObject loader = Py.None;
         PySystemState sys = Py.getSystemState();
         PyObject metaPath = sys.meta_path;
 
+        // Check for importers along sys.meta_path
         for (PyObject importer : metaPath.asIterable()) {
             PyObject findModule = importer.__getattr__("find_module");
             loader = findModule.__call__(new PyObject[] { //
@@ -523,23 +693,27 @@
             }
         }
 
+        // Attempt to load from (prepared) builtins in sys.builtins.
         PyObject ret = loadBuiltin(moduleName);
         if (ret != null) {
             return ret;
         }
 
+        // Note the path here may be sys.path or the search path of a Python package.
         path = path == null ? sys.path : path;
         for (int i = 0; i < path.__len__(); i++) {
             PyObject p = path.__getitem__(i);
+            // Is there a path-specific importer?
             PyObject importer = getPathImporter(sys.path_importer_cache, sys.path_hooks, p);
             if (importer != Py.None) {
+                // A specific importer is defined. Try its finder method.
                 PyObject findModule = importer.__getattr__("find_module");
                 loader = findModule.__call__(new PyObject[] {new PyString(moduleName)});
                 if (loader != Py.None) {
                     return loadFromLoader(loader, moduleName);
                 }
             }
-            // p could be unicode or bytes (in the file system encoding)
+            // p could be a unicode or bytes object (in the file system encoding)
             ret = loadFromSource(sys, name, moduleName, Py.fileSystemDecode(p));
             if (ret != null) {
                 return ret;
@@ -549,6 +723,14 @@
         return ret;
     }
 
+    /**
+     * Load a built-in module by reference to {@link PySystemState#builtins}, which maps Python
+     * module names to class names. Special treatment is given to the modules {@code sys} and
+     * {@code __builtin__}.
+     *
+     * @param fully-qualified name of module
+     * @return
+     */
     private static PyObject loadBuiltin(String name) {
         if (name == "sys") {
             Py.writeComment(IMPORT_LOG, "'" + name + "' as sys in builtin modules");
@@ -569,7 +751,8 @@
                     }
                     return createFromClass(name, c);
                 } catch (NoClassDefFoundError e) {
-                    throw Py.ImportError("Cannot import " + name + ", missing class " + c.getName());
+                    throw Py.ImportError(
+                            "Cannot import " + name + ", missing class " + c.getName());
                 }
             }
         }
@@ -593,39 +776,54 @@
         return createFromPyClass(name, stream, false, sourceName, compiledName);
     }
 
-    static PyObject loadFromSource(PySystemState sys, String name, String modName, String entry) {
-        String dirName = sys.getPath(entry);
+    /**
+     * Import a module defined in Python by loading it from source (or a compiled
+     * {@code name$pyclass}) file in the specified location (often an entry from {@code sys.path},
+     * or a sub-directory of it named for the {@code modName}. For example, if {@code name} is
+     * "pkg1" and the {@code modName} is "pkg.pkg1", {@code location} might be "mylib/pkg".
+     *
+     * @param sys the sys module of the interpreter importing the module.
+     * @param name by which to look for files or a directory representing the module.
+     * @param modName name by which to enter the module in {@code sys.modules}.
+     * @param location where to look for the {@code name}.
+     * @return the module if we can load it (or {@code null} if we can't).
+     */
+    static PyObject loadFromSource(PySystemState sys, String name, String modName,
+            String location) {
+        String dirName = sys.getPath(location);
         String sourceName = "__init__.py";
         String compiledName = makeCompiledFilename(sourceName);
         // display names are for identification purposes (e.g. __file__): when entry is
         // null it forces java.io.File to be a relative path (e.g. foo/bar.py instead of
         // /tmp/foo/bar.py)
-        String displayDirName = entry.equals("") ? null : entry;
+        String displayDirName = location.equals("") ? null : location;
         String displaySourceName = new File(new File(displayDirName, name), sourceName).getPath();
         String displayCompiledName =
                 new File(new File(displayDirName, name), compiledName).getPath();
 
-        // First check for packages
-        File dir = new File(dirName, name);
-        File sourceFile = new File(dir, sourceName);
-        File compiledFile = new File(dir, compiledName);
+        // Create file objects to check for a Python package
+        File dir = new File(dirName, name);                 // entry/name/
+        File sourceFile = new File(dir, sourceName);        // entry/name/__init__.py
+        File compiledFile = new File(dir, compiledName);    // entry/name/__init__$py.class
 
-        boolean pkg = false;
+        boolean isPackage = false;
         try {
             if (dir.isDirectory()) {
                 if (caseok(dir, name) && (sourceFile.isFile() || compiledFile.isFile())) {
-                    pkg = true;
+                    isPackage = true;
                 } else {
                     String printDirName = PyString.encode_UnicodeEscape(dir.getPath(), '\'');
                     Py.warning(Py.ImportWarning, String.format(
                             "Not importing directory %s: missing __init__.py", printDirName));
+                    return null;
                 }
             }
         } catch (SecurityException e) {
             // ok
         }
 
-        if (!pkg) {
+        if (!isPackage) {
+            // The source is entry/name.py and compiled is entry/name$py.class
             Py.writeDebug(IMPORT_LOG, "trying source " + dir.getPath());
             sourceName = name + ".py";
             compiledName = makeCompiledFilename(sourceName);
@@ -634,18 +832,24 @@
             sourceFile = new File(dirName, sourceName);
             compiledFile = new File(dirName, compiledName);
         } else {
+            // Create a PyModule (uninitialised) for name.py, called modName in sys.modules
             PyModule m = addModule(modName);
             PyObject filename = Py.newStringOrUnicode(new File(displayDirName, name).getPath());
+            // FIXME: FS encode file name for __path__
             m.__dict__.__setitem__("__path__", new PyList(new PyObject[] {filename}));
         }
 
+        // Load and execute the module, from its compiled or source form.
         try {
+            // Try to create the module from source or an existing compiled class.
             if (sourceFile.isFile() && caseok(sourceFile, sourceName)) {
                 long pyTime = sourceFile.lastModified();
                 if (compiledFile.isFile() && caseok(compiledFile, compiledName)) {
+                    // We have the compiled file and will use that if it is not out of date
                     Py.writeDebug(IMPORT_LOG, "trying precompiled " + compiledFile.getPath());
                     long classTime = compiledFile.lastModified();
                     if (classTime >= pyTime) {
+                        // The compiled file does not appear out of date relative to the source.
                         PyObject ret = createFromPyClass(modName, makeStream(compiledFile), //
                                 true, // OK to fail here as we have the source
                                 displaySourceName, displayCompiledName, pyTime);
@@ -653,15 +857,15 @@
                             return ret;
                         }
                     }
-                    return createFromSource(modName, makeStream(sourceFile), displaySourceName,
-                            compiledFile.getPath(), pyTime);
                 }
+                // The compiled class is not present, was out of date, or trying to use it failed.
                 return createFromSource(modName, makeStream(sourceFile), displaySourceName,
                         compiledFile.getPath(), pyTime);
             }
 
-            // If no source, try loading precompiled
-            Py.writeDebug(IMPORT_LOG, "trying precompiled with no source " + compiledFile.getPath());
+            // If no source, try loading compiled
+            Py.writeDebug(IMPORT_LOG,
+                    "trying precompiled with no source " + compiledFile.getPath());
             if (compiledFile.isFile() && caseok(compiledFile, compiledName)) {
                 return createFromPyClass(modName, makeStream(compiledFile), //
                         false, // throw ImportError here if this fails
@@ -673,6 +877,28 @@
         return null;
     }
 
+    /**
+     * Check that the canonical name of {@code file} matches {@code filename}, case-sensitively,
+     * even when the OS platform is case-insensitive. This is used to obtain as a check during
+     * import on platforms (Windows) that may be case-insensitive regarding file open. It is assumed
+     * that {@code file} was derived from attempting to find {@code filename}, so it returns
+     * {@code true} on a case-sensitive platform.
+     * <p>
+     * Algorithmically, we return {@code true} if any of the following is true:
+     * <ul>
+     * <li>{@link Options#caseok} is {@code true} (default is {@code false}).</li>
+     * <li>The platform is case sensitive (according to
+     * {@link PlatformUtil#isCaseInsensitive()})</li>
+     * <li>The name part of the canonical path of {@code file} starts with {@code filename}</li>
+     * <li>The name of any sibling (in the same directory as) {@code file} equals
+     * {@code filename}</li>
+     * </ul>
+     * and false otherwise.
+     *
+     * @param file to be tested
+     * @param filename to be matched
+     * @return {@code file} matches {@code filename}
+     */
     public static boolean caseok(File file, String filename) {
         if (Options.caseok || !PlatformUtil.isCaseInsensitive()) {
             return true;
@@ -681,13 +907,8 @@
             File canFile = new File(file.getCanonicalPath());
             boolean match = filename.regionMatches(0, canFile.getName(), 0, filename.length());
             if (!match) {
-                // possibly a symlink. Get parent and look for exact match in listdir()
-                // This is what CPython does in the case of Mac OS X and Cygwin.
-                // XXX: This will be a performance hit, maybe jdk7 nio2 can give us a better
-                // method?
-                File parent = file.getParentFile();
-                String[] children = parent.list();
-                for (String c : children) {
+                // Get parent and look for exact match in listdir(). This is horrible, but rare.
+                for (String c : file.getParentFile().list()) {
                     if (c.equals(filename)) {
                         return true;
                     }
@@ -718,17 +939,17 @@
 
     /**
      * Find the parent package name for a module.
-     *
+     * <p>
      * If __name__ does not exist in the module or if level is <code>0</code>, then the parent is
      * <code>null</code>. If __name__ does exist and is not a package name, the containing package
      * is located. If no such package exists and level is <code>-1</code>, the parent is
      * <code>null</code>. If level is <code>-1</code>, the parent is the current name. Otherwise,
-     * <code>level-1</code> doted parts are stripped from the current name. For example, the
+     * <code>level-1</code> dotted parts are stripped from the current name. For example, the
      * __name__ <code>"a.b.c"</code> and level <code>2</code> would return <code>"a.b"</code>, if
      * <code>c</code> is a package and would return <code>"a"</code>, if <code>c</code> is not a
      * package.
      *
-     * @param dict the __dict__ of a loaded module
+     * @param dict the __dict__ of a loaded module that is the context of the import statement
      * @param level used for relative and absolute imports. -1 means try both, 0 means absolute
      *            only, positive ints represent the level to look upward for a relative path (1
      *            means current package, 2 means one level up). See PEP 328 at
@@ -745,7 +966,7 @@
             return null;
         }
 
-        PyObject tmp = dict.__finditem__("__package__");
+        PyObject tmp = dict.__finditem__("__package__"); // XXX: why is this not guaranteed set?
         if (tmp != null && tmp != Py.None) {
             if (!Py.isInstance(tmp, PyString.TYPE)) {
                 throw Py.ValueError("__package__ set to non-string");
@@ -806,128 +1027,220 @@
     }
 
     /**
+     * Try to import the module named by <i>parentName.name</i>. The method tries 3 ways, accepting
+     * the first that * succeeds:
+     * <ol>
+     * <li>Check for the module (by its fully-qualified name) in {@code sys.modules}.</li>
+     * <li>If {@code mod==null}, try to load the module via
+     * {@link #find_module(String, String, PyList)}. If {@code mod!=null}, find it as an attribute
+     * of {@code mod} via its {@link PyObject#impAttr(String)} method (which then falls back to
+     * {@code find_module} if {@code mod} has a {@code __path__}). Either way, add the loaded module
+     * to {@code sys.modules}.</li>
+     * <li>Try to load the module as a Java package by the name {@code outerFullName}
+     * {@link JavaImportHelper#tryAddPackage(String, PyObject)}.</li>
+     * </ol>
+     * Finally, if one is found, If a module by the given name already exists in {@code sys.modules}
+     * it will be returned from there directly. Otherwise, in {@code mod==null} (frequent case) it
+     * will be looked for via {@link #find_module(String, String, PyList)}.
+     * <p>
+     * The case {@code mod!=null} supports circumstances in which the module sought may be found as
+     * an attribute of a parent module.
      *
-     * @param mod a previously loaded module
-     * @param parentNameBuffer
-     * @param name the name of the module to load
-     * @return null or None
+     * @param mod if not {@code null}, a package where the module may be an attribute.
+     * @param parentName parent name of the module. (Buffer is appended with "." and {@code name}.
+     * @param name the (simple) name of the module to load
+     * @param outerFullName name to use with the {@code JavaImportHelper}.
+     * @param fromlist if not {@code null} the import is {@code from <module> import <fromlist>}
+     * @return the imported module (or {@code null} or {@link Py#None} on failure).
      */
-    private static PyObject import_next(PyObject mod, StringBuilder parentNameBuffer, String name,
+    private static PyObject import_next(PyObject mod, StringBuilder parentName, String name,
             String outerFullName, PyObject fromlist) {
-        if (parentNameBuffer.length() > 0 && name != null && name.length() > 0) {
-            parentNameBuffer.append('.');
+
+        // Concatenate the parent name and module name *modifying the parent name buffer*
+        if (parentName.length() > 0 && name != null && name.length() > 0) {
+            parentName.append('.');
         }
-        parentNameBuffer.append(name);
+        String fullName = parentName.append(name).toString().intern();
 
-        String fullName = parentNameBuffer.toString().intern();
-
+        // Check if already in sys.modules (possibly Py.None).
         PyObject modules = Py.getSystemState().modules;
         PyObject ret = modules.__finditem__(fullName);
         if (ret != null) {
             return ret;
         }
+
         if (mod == null) {
+            // We are looking for a top-level module
             ret = find_module(fullName, name, null);
         } else {
+            // Look within mod as enclosing package
             ret = mod.impAttr(name.intern());
         }
+
         if (ret == null || ret == Py.None) {
+            // Maybe this is a Java package: full name from the import and maybe classes to import
             if (JavaImportHelper.tryAddPackage(outerFullName, fromlist)) {
+                // The action has already added it to sys.modules
                 ret = modules.__finditem__(fullName);
             }
             return ret;
         }
+
+        // The find operation may have added to sys.modules the module object we seek.
         if (modules.__finditem__(fullName) == null) {
-            modules.__setitem__(fullName, ret);
+            modules.__setitem__(fullName, ret);     // Nope, add it
         } else {
-            ret = modules.__finditem__(fullName);
+            ret = modules.__finditem__(fullName);   // Yep, return that instead
         }
+
+        // On OSX we currently have to monkeypatch setuptools.command.easy_install.
         if (IS_OSX && fullName.equals("setuptools.command")) {
-            // On OSX we currently have to monkeypatch setuptools.command.easy_install.
             // See http://bugs.jython.org/issue2570
             load("_fix_jython_setuptools_osx");
         }
+
         return ret;
     }
 
-    // never returns null or None
-    private static PyObject import_first(String name, StringBuilder parentNameBuffer) {
-        PyObject ret = import_next(null, parentNameBuffer, name, null, null);
+    /**
+     * Top of the import logic in the case of a simple {@code import a.b.c.m}.
+     *
+     * @param name fully-qualified name of module to import {@code import a.b.c.m}
+     * @param parentName used as a workspace as the search descends the package hierarchy
+     * @return the named module (never {@code null} or {@code None})
+     * @throws PyException (ImportError) if not found
+     */
+    private static PyObject import_first(String name, StringBuilder parentName) throws PyException {
+        PyObject ret = import_next(null, parentName, name, null, null);
         if (ret == null || ret == Py.None) {
             throw Py.ImportError("No module named " + name);
         }
         return ret;
     }
 
-    private static PyObject import_first(String name, StringBuilder parentNameBuffer,
-            String fullName, PyObject fromlist) {
-        PyObject ret = import_next(null, parentNameBuffer, name, fullName, fromlist);
+    /**
+     * Top of the import logic in the case of a complex {@code from a.b.c.m import n1, n2, n3}.
+     *
+     * @param name fully-qualified name of module to import {@code a.b.c.m}.
+     * @param parentName used as a workspace as the search descends the package hierarchy
+     * @param fullName the "outer" name by which the module is known {@code a.b.c.m}.
+     * @param fromlist names to import from the module {@code n1, n2, n3}.
+     * @return the named module (never returns {@code null} or {@code None})
+     * @throws PyException (ImportError) if not found
+     */
+    private static PyObject import_first(String name, StringBuilder parentName,
+            String fullName, PyObject fromlist) throws PyException {
+
+        // Try the "normal" Python import process
+        PyObject ret = import_next(null, parentName, name, fullName, fromlist);
+
+        // If unsuccessful try importing as a Java package
         if (ret == null || ret == Py.None) {
             if (JavaImportHelper.tryAddPackage(fullName, fromlist)) {
-                ret = import_next(null, parentNameBuffer, name, fullName, fromlist);
+                ret = import_next(null, parentName, name, fullName, fromlist);
             }
         }
+
+        // If still unsuccessful, it's an error
         if (ret == null || ret == Py.None) {
             throw Py.ImportError("No module named " + name);
         }
         return ret;
     }
 
-    // Hierarchy-recursively search for dotted name in mod;
-    // never returns null or None
+    /**
+     * Iterate through the components (after the first) of a fully-qualified module name
+     * {@code a.b.c.m} finding the corresponding modules {@code a.b}, {@code a.b.c}, and
+     * {@code a.b.c.m}, importing them if necessary. This is a helper to
+     * {@link #import_module_level(String, boolean, PyObject, PyObject, int)}, used when the module
+     * name involves more than one level.
+     * <p>
+     * This method may be called in support of (effectively) of a simple import statement like
+     * {@code import a.b.c.m} or a complex statement {@code from a.b.c.m import n1, n2, n3}. This
+     * method always returns the "deepest" name, in the example, the module {@code m} whose full
+     * name is {@code a.b.c.m}.
+     *
+     * @param mod top module of the import
+     * @param parentName used as a workspace as the search descends the package hierarchy
+     * @param restOfName {@code b.c.m}
+     * @param fullName {@code a.b.c.m}
+     * @param fromlist names to import from the module {@code n1, n2, n3}.
+     * @return the last named module (never {@code null} or {@code None})
+     * @throws PyException (ImportError) if not found
+     */
     // ??pending: check if result is really a module/jpkg/jclass?
-    private static PyObject import_logic(PyObject mod, StringBuilder parentNameBuffer,
-            String dottedName, String fullName, PyObject fromlist) {
+    private static PyObject import_logic(PyObject mod, StringBuilder parentName, String restOfName,
+            String fullName, PyObject fromlist) throws PyException {
+
         int dot = 0;
-        int last_dot = 0;
+        int start = 0;
 
         do {
+            // Extract the name that starts at restOfName[start:] up to next dot.
             String name;
-            dot = dottedName.indexOf('.', last_dot);
+            dot = restOfName.indexOf('.', start);
             if (dot == -1) {
-                name = dottedName.substring(last_dot);
+                name = restOfName.substring(start);
             } else {
-                name = dottedName.substring(last_dot, dot);
+                name = restOfName.substring(start, dot);
             }
+
             PyJavaPackage jpkg = null;
             if (mod instanceof PyJavaPackage) {
-                jpkg = (PyJavaPackage)mod;
+                jpkg = (PyJavaPackage) mod;
             }
 
-            mod = import_next(mod, parentNameBuffer, name, fullName, fromlist);
+            // Find (and maybe import) the package/module corresponding to this new segment.
+            mod = import_next(mod, parentName, name, fullName, fromlist);
+
+            // Try harder when importing as a Java package :/
             if (jpkg != null && (mod == null || mod == Py.None)) {
                 // try again -- under certain circumstances a PyJavaPackage may
                 // have been added as a side effect of the last import_next
                 // attempt. see Lib/test_classpathimport.py#test_bug1126
-                mod = import_next(jpkg, parentNameBuffer, name, fullName, fromlist);
+                mod = import_next(jpkg, parentName, name, fullName, fromlist);
             }
+
+            // If still unsuccessful, it's an error
             if (mod == null || mod == Py.None) {
                 throw Py.ImportError("No module named " + name);
             }
-            last_dot = dot + 1;
+
+            // Next module/package simple-name starts just after the last dot we found
+            start = dot + 1;
         } while (dot != -1);
 
         return mod;
     }
 
     /**
-     * @param name
-     * @param top
-     * @param modDict
-     * @return a module
+     * Import a module by name. This supports the default {@code __import__()} function
+     * {@code __builtin__.__import__}. (Called with the import system locked.)
+     *
+     * @param name qualified name of the package/module to import (may be relative)
+     * @param top if true, return the top module in the name, otherwise the last
+     * @param modDict the __dict__ of the importing module (used to navigate a relative import)
+     * @param fromlist list of names being imported
+     * @param level 0=absolute, n&gt;0=relative levels to go up, -1=try relative then absolute.
+     * @return an imported module (Java or Python)
      */
     private static PyObject import_module_level(String name, boolean top, PyObject modDict,
             PyObject fromlist, int level) {
+
+        // Check for basic invalid call
         if (name.length() == 0) {
             if (level == 0 || modDict == null) {
                 throw Py.ValueError("Empty module name");
             } else {
                 PyObject moduleName = modDict.__findattr__("__name__");
+                // XXX: should this test be for "__main__"?
                 if (moduleName != null && moduleName.toString().equals("__name__")) {
                     throw Py.ValueError("Attempted relative import in non-package");
                 }
             }
         }
+
+        // Seek the module (in sys.modules) that the import is relative to.
         PyObject modules = Py.getSystemState().modules;
         PyObject pkgMod = null;
         String pkgName = null;
@@ -938,6 +1251,8 @@
                 pkgMod = null;
             }
         }
+
+        // Extract the first element of the (fully qualified) name.
         int dot = name.indexOf('.');
         String firstName;
         if (dot == -1) {
@@ -945,22 +1260,31 @@
         } else {
             firstName = name.substring(0, dot);
         }
-        StringBuilder parentNameBuffer = new StringBuilder(pkgMod != null ? pkgName : "");
-        PyObject topMod = import_next(pkgMod, parentNameBuffer, firstName, name, fromlist);
+
+        // Import the first-named module, relative to pkgMod (which may be null)
+        StringBuilder parentName = new StringBuilder(pkgMod != null ? pkgName : "");
+        PyObject topMod = import_next(pkgMod, parentName, firstName, name, fromlist);
+
         if (topMod == Py.None || topMod == null) {
-            parentNameBuffer = new StringBuilder("");
+            // The first attempt failed. This time
+            parentName = new StringBuilder("");
             // could throw ImportError
             if (level > 0) {
-                topMod = import_first(pkgName + "." + firstName, parentNameBuffer, name, fromlist);
+                // Import relative to context. pkgName was already computed from level.
+                topMod = import_first(pkgName + "." + firstName, parentName, name, fromlist);
             } else {
-                topMod = import_first(firstName, parentNameBuffer, name, fromlist);
+                // Absolute import
+                topMod = import_first(firstName, parentName, name, fromlist);
             }
         }
+
         PyObject mod = topMod;
+
         if (dot != -1) {
-            // could throw ImportError
-            mod = import_logic(topMod, parentNameBuffer, name.substring(dot + 1), name, fromlist);
+            // This is a dotted name: work through the remaining name elements.
+            mod = import_logic(topMod, parentName, name.substring(dot + 1), name, fromlist);
         }
+
         if (top) {
             return topMod;
         }
@@ -990,10 +1314,28 @@
         }
     }
 
+    /**
+     * Ensure that the items mentioned in the from-list of an import are actually present, even if
+     * they are modules we have not imported yet.
+     *
+     * @param mod module we are importing from
+     * @param fromlist tuple of names to import
+     * @param name of module we are importing from (as given, may be relative)
+     */
     private static void ensureFromList(PyObject mod, PyObject fromlist, String name) {
         ensureFromList(mod, fromlist, name, false);
     }
 
+    /**
+     * Ensure that the items mentioned in the from-list of an import are actually present, even if
+     * they are modules we have not imported yet.     *
+     *
+     * @param mod module we are importing from
+     * @param fromlist tuple of names to import
+     * @param name of module we are importing from (as given, may be relative)
+     * @param recursive if true, when the from-list includes "*", do not import __all__
+     */
+    // XXX: effect of the recursive argument is hard to fathom.
     private static void ensureFromList(PyObject mod, PyObject fromlist, String name,
             boolean recursive) {
         // This can happen with imports like "from . import foo"
@@ -1044,11 +1386,14 @@
     }
 
     /**
-     * Import a module by name. This is the default call for __builtin__.__import__.
+     * Import a module by name. This supports the default {@code __import__()} function
+     * {@code __builtin__.__import__}. Locks the import system while it operates.
      *
-     * @param name the name of the package to import
+     * @param name the fully-qualified name of the package/module to import
      * @param top if true, return the top module in the name, otherwise the last
-     * @param modDict the __dict__ of an already imported module
+     * @param modDict the __dict__ of the importing module (used for name in relative import)
+     * @param fromlist list of names being imported
+     * @param level 0=absolute, n&gt;0=relative levels to go up, -1=try relative then absolute.
      * @return an imported module (Java or Python)
      */
     public static PyObject importName(String name, boolean top, PyObject modDict,

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Thu Mar  7 18:10:45 2019
From: jython-checkins at python.org (jeff.allen)
Date: Thu, 07 Mar 2019 23:10:45 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Attribute_access_is_import?=
 =?utf-8?q?_*only*_for_Java_packages_and_classes_=28fixes_=232654=29=2E?=
Message-ID: <20190307231045.1.150A41D0DA209548@mg.python.org>

https://hg.python.org/jython/rev/22c19e5a77ad
changeset:   8221:22c19e5a77ad
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Thu Mar 07 09:02:07 2019 +0000
summary:
  Attribute access is import *only* for Java packages and classes (fixes #2654).

Refactors PyModule.impAttr and PyModule.__findattr_ex__ so that the latter
only looks for Java packages and classes, rather than invoking a full impAttr,
which would import Python modules.

An addition to test_import_jy tests exercises a range of import patterns,
each time in a clean subprocess, covering combinations of Python and Java
in pure and mixed hierarchy.

files:
  Lib/test/test_import_jy.py        |  224 +++++++++++++++++-
  Lib/test/test_import_jy.zip       |  Bin 
  src/org/python/core/PyModule.java |  132 +++++++---
  src/org/python/core/PyObject.java |   10 +-
  src/org/python/core/imp.java      |   31 +-
  5 files changed, 333 insertions(+), 64 deletions(-)


diff --git a/Lib/test/test_import_jy.py b/Lib/test/test_import_jy.py
--- a/Lib/test/test_import_jy.py
+++ b/Lib/test/test_import_jy.py
@@ -11,8 +11,10 @@
 import tempfile
 import unittest
 import subprocess
+import zipfile
 from test import test_support
 from test_chdir import read, safe_mktemp, COMPILED_SUFFIX
+from doctest import script_from_examples
 
 class MislabeledImportTestCase(unittest.TestCase):
 
@@ -233,11 +235,225 @@
         self.assertEqual(cm.exception.reason, "ordinal not in range(128)")
 
 
+class MixedImportTestCase(unittest.TestCase):
+    #
+    # This test case depends on material in a file structure unpacked
+    # from an associated ZIP archive. The test depends on Python source
+    # and Java class files. The archive also contains the Java source
+    # from which the class files may be regenerated if necessary.
+    #
+    # To regenerate the class files, explode the archive in a
+    # convenient spot on the file system and compile them with javac at
+    # the lowest supported code standard (currently Java 7), e.g. (posh)
+    #   PS jython-trunk> cd mylib
+    #   PS mylib> javac  $(get-childitem -Recurse -Name -Include "*.java")
+    # or the equivalent Unix command using find.
+
+    ZIP = test_support.findfile("test_import_jy.zip")
+
+    @classmethod
+    def setUpClass(cls):
+        td = tempfile.mkdtemp()
+        cls.source = os.path.join(td, "test.py")
+        cls.setpath = "import sys; sys.modules[0] = r'" + td + "'"
+        zip = zipfile.ZipFile(cls.ZIP, 'r')
+        zip.extractall(td)
+        cls.tmpdir = td
+
+    @classmethod
+    def tearDownClass(cls):
+        td = cls.tmpdir
+        if td and os.path.isdir(td):
+            test_support.rmtree(td)
+
+    def make_prog(self, *script):
+        "Write a program to test.py"
+        with open(self.source, "wt") as f:
+            print >> f, MixedImportTestCase.setpath
+            for line in script:
+                print >> f, line
+            print >> f, "raise SystemExit"
+
+    def run_prog(self):
+        # Feed lines to interpreter and capture output
+        process = subprocess.Popen([sys.executable, "-S", MixedImportTestCase.source],
+                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        output, err = process.communicate()
+        retcode = process.poll()
+        if retcode:
+            raise subprocess.CalledProcessError(retcode, sys.executable, output=err)
+        return output
+
+    def module_regex(self, module):
+        "Partial filename from module"
+        sep = "\\" + os.path.sep
+        return sep + module.replace('.', sep)
+
+    def check_package(self, line, module):
+        target = "Executed: .*" +  self.module_regex(module) + "\\" + os.path.sep \
+                    + r"__init__(\.py|\$py\.class|\.pyc)"
+        self.assertRegexpMatches(line, target)
+
+    def check_module(self, line, module):
+        "Check output from loading a module"
+        target = "Executed: .*" + self.module_regex(module) + r"(\.py|\$py\.class|\.pyc)"
+        self.assertRegexpMatches(line, target)
+
+    def test_import_to_program(self):
+        # A Python module in a Python program
+        self.make_prog("import a.b.c.m", "print repr(a.b.c.m)")
+        out = self.run_prog().splitlines()
+        self.check_package(out[0], "a")
+        self.check_package(out[1], "a.b")
+        self.check_package(out[2], "a.b.c")
+        self.check_module(out[3], "a.b.c.m")
+        self.assertRegexpMatches(out[4], r"\<module 'a\.b\.c\.m' from .*\>")
+
+    def test_import_to_program_no_magic(self):
+        # A Python module in a Python program (issue 2654)
+        self.make_prog("import a.b, a.b.c", "print repr(a.b.m3)")
+        try:
+            out = self.run_prog()
+            self.fail("reference to a.b.m3 did not raise exception")
+        except subprocess.CalledProcessError as e:
+            self.assertRegexpMatches(e.output, r"AttributeError: .* has no attribute 'm3'")
+
+    def test_import_relative_implicit(self):
+        # A Python module by implicit relative import (no dots)
+        self.make_prog("import a.b.m3")
+        out = self.run_prog().splitlines()
+        self.check_package(out[0], "a")
+        self.check_package(out[1], "a.b")
+        self.check_module(out[2], "a.b.m3")
+        self.check_package(out[3], "a.b.c")
+        self.check_module(out[4], "a.b.c.m")
+        self.assertRegexpMatches(out[5], r"\<module 'a\.b\.c\.m' from .*\>")
+
+    def test_import_absolute_implicit(self):
+        # A built-in module by absolute import (but relative must be tried first)
+        self.make_prog("import a.b.m4")
+        out = self.run_prog().splitlines()
+        self.check_package(out[0], "a")
+        self.check_package(out[1], "a.b")
+        self.check_module(out[2], "a.b.m4")
+        self.assertRegexpMatches(out[3], r"\<module 'sys' \(built-in\)\>")
+
+    def test_import_from_module(self):
+        # A Python module by from-import
+        self.make_prog("import a.b.m5")
+        out = self.run_prog().splitlines()
+        self.check_package(out[0], "a")
+        self.check_package(out[1], "a.b")
+        self.check_module(out[2], "a.b.m5")
+        self.check_package(out[3], "a.b.c")
+        self.check_module(out[4], "a.b.c.m")
+        self.assertRegexpMatches(out[5], r"\<module 'a\.b\.c\.m' from .*\>")
+        self.assertRegexpMatches(out[6], r"1 2")
+
+    def test_import_from_relative_module(self):
+        # A Python module by relative from-import
+        self.make_prog("import a.b.m6")
+        out = self.run_prog().splitlines()
+        self.check_package(out[0], "a")
+        self.check_package(out[1], "a.b")
+        self.check_module(out[2], "a.b.m6")
+        self.check_package(out[3], "a.b.c")
+        self.check_module(out[4], "a.b.c.m")
+        self.assertRegexpMatches(out[5], r"\<module 'a\.b\.c\.m' from .*\>")
+        self.assertRegexpMatches(out[6], r"1 2")
+
+    def check_java_package(self, line, module):
+        target = r"\<java package " + module + r" 0x[0-9a-f]+\>"
+        self.assertRegexpMatches(line, target)
+
+    def test_import_java_java(self):
+        # A Java class in a Java package by from-import
+        self.make_prog("from jpkg.j import K", "print repr(K)")
+        out = self.run_prog().splitlines()
+        self.assertRegexpMatches(out[0], r"\<type 'jpkg.j.K'\>")
+
+    def test_import_java_java_magic(self):
+        # A Java class in a Java package
+        # with implicit sub-module and class import
+        self.make_prog(
+            "import jpkg",
+            "print repr(jpkg)",
+            "print repr(jpkg.j)",
+            "print repr(jpkg.j.K)",
+            "print repr(jpkg.L)")
+        out = self.run_prog().splitlines()
+        self.check_java_package(out[0], "jpkg")
+        self.check_java_package(out[1], "jpkg.j")
+        self.assertRegexpMatches(out[2], r"\<type 'jpkg.j.K'\>")
+        self.assertRegexpMatches(out[3], r"\<type 'jpkg.L'\>")
+
+    def test_import_java_python(self):
+        # A Java class in a Python package by from-import
+        self.make_prog("from mix.b import K1", "print repr(K1)")
+        out = self.run_prog().splitlines()
+        self.check_package(out[0], "mix")
+        self.check_package(out[1], "mix.b")
+        self.assertRegexpMatches(out[2], r"\<type 'mix.b.K1'\>")
+
+    def test_import_java_python_magic(self):
+        # A Java class in a Python package
+        # with implicit sub-module and class import
+        self.make_prog(
+            "import mix",
+            "print repr(mix.b)",
+            "print repr(mix.b.K1)",
+            "import mix.b",
+            "print repr(mix.b)",
+            "print repr(mix.b.K1)",
+            "print repr(mix.J1)")
+        out = self.run_prog().splitlines()
+        self.check_package(out[0], "mix")
+        self.check_java_package(out[1], "mix.b")
+        self.assertRegexpMatches(out[2], r"\<type 'mix.b.K1'\>")
+        self.check_package(out[3], "mix.b")
+        self.assertRegexpMatches(out[4], r"\<module 'mix\.b' from .*\>")
+        self.assertRegexpMatches(out[5], r"\<type 'mix.b.K1'\>")
+        self.assertRegexpMatches(out[6], r"\<type 'mix.J1'\>")
+
+    def test_import_javapkg_python(self):
+        # A Java package in a Python package
+        self.make_prog("import mix.j", "print repr(mix.j)", "print repr(mix.j.K2)")
+        out = self.run_prog().splitlines()
+        self.check_package(out[0], "mix")
+        self.check_java_package(out[1], "mix.j")
+        self.assertRegexpMatches(out[2], r"\<type 'mix.j.K2'\>")
+
+    def test_import_java_from_javapkg(self):
+        # A Java class in a Java package in a Python package
+        self.make_prog("from mix.j import K2", "print repr(K2)")
+        out = self.run_prog().splitlines()
+        self.check_package(out[0], "mix")
+        self.assertRegexpMatches(out[1], r"\<type 'mix.j.K2'\>")
+
+    def test_import_javapkg_magic(self):
+        # A Java class in a Java package in a Python package
+        # with implicit sub-module and class import
+        self.make_prog(
+            "import mix",
+            "print repr(mix.J1)",
+            "print repr(mix.j)",
+            "print repr(mix.j.K2)",
+            )
+        out = self.run_prog().splitlines()
+        self.check_package(out[0], "mix")
+        self.assertRegexpMatches(out[1], r"\<type 'mix.J1'\>")
+        self.check_java_package(out[2], "mix.j")
+        self.assertRegexpMatches(out[3], r"\<type 'mix.j.K2'\>")
+
+
 def test_main():
-    test_support.run_unittest(MislabeledImportTestCase,
-                              OverrideBuiltinsImportTestCase,
-                              ImpTestCase,
-                              UnicodeNamesTestCase)
+    test_support.run_unittest(
+            MislabeledImportTestCase,
+            OverrideBuiltinsImportTestCase,
+            ImpTestCase,
+            UnicodeNamesTestCase,
+            MixedImportTestCase
+    )
 
 if __name__ == '__main__':
     test_main()
diff --git a/Lib/test/test_import_jy.zip b/Lib/test/test_import_jy.zip
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..b76f432078589f7dfcdd14269b75179f32a8af4e
GIT binary patch
[stripped]
diff --git a/src/org/python/core/PyModule.java b/src/org/python/core/PyModule.java
--- a/src/org/python/core/PyModule.java
+++ b/src/org/python/core/PyModule.java
@@ -91,54 +91,100 @@
     /**
      * {@inheritDoc}
      * <p>
-     * Overridden in {@code PyModule} to search for the named attribute as a
-     * module in {@code sys.modules} (using the key {@code ".".join(self.__name__, name)}) and on the
-     * {@code self.__path__}.
+     * Overridden in {@code PyModule} to search for a sub-module of this module (using the key
+     * {@code ".".join(self.__name__, name)}) in {@code sys.modules}, on the {@code self.__path__},
+     * and as a Java package with the same name. The named sub-module becomes an attribute of this
+     * module (in {@code __dict__}).
      */
     @Override
     protected PyObject impAttr(String name) {
-        // Get hold of the module dictionary and __name__, and __path__ if it has one.
-        if (__dict__ == null || name.length() == 0) {
-            return null;
-        }
-        PyObject path = __dict__.__finditem__("__path__");
-        if (path == null) {
-            path = new PyList();
+
+        // Some of our look-up needs the full name, deduced from __name__ and name.
+        String fullName = getFullName(name);
+
+        if (fullName != null) {
+            // Maybe the attribute is a Python sub-module
+            PyObject attr = findSubModule(name, fullName);
+
+            // Or is a Java package
+            if (attr == null) {
+                // Still looking: maybe it's a Java package?
+                attr = PySystemState.packageManager.lookupName(fullName);
+            }
+
+            // Add as an attribute the thing we found (if not still null)
+            return addedSubModule(name, fullName, attr);
         }
-        PyObject pyName = __dict__.__finditem__("__name__");
-        if (pyName == null) {
-            return null;
-        }
+        return null;
+    }
 
-        // Maybe the module we're looking for is in sys.modules
-        String fullName = (pyName.__str__().toString() + '.' + name).intern();
-        PyObject modules = Py.getSystemState().modules;
-        PyObject attr = modules.__finditem__(fullName);
-
-        // If not, look along the module's __path__
-        if (path instanceof PyList) {
+    /**
+     * Find Python sub-module within this object, within {@code sys.modules} or along this module's
+     * {@code __path__}.
+     *
+     * @param name simple name of sub package
+     * @param fullName of sub package
+     * @return module found or {@code null}
+     */
+    private PyObject findSubModule(String name, String fullName) {
+        PyObject attr =  null;
+        if (fullName != null) {
+            // The module may already have been loaded in sys.modules
+            attr = Py.getSystemState().modules.__finditem__(fullName);
+            // Or it may be found as a Python module along this module's __path__
             if (attr == null) {
-                attr = imp.find_module(name, fullName, (PyList)path);
+                PyObject path = __dict__.__finditem__("__path__");
+                if (path == null) {
+                    attr = imp.find_module(name, fullName, new PyList());
+                } else if (path instanceof PyList) {
+                    attr = imp.find_module(name, fullName, (PyList) path);
+                } else if (path != Py.None) {
+                    throw Py.TypeError("__path__ must be list or None");
+                }
             }
-        } else if (path != Py.None) {
-            throw Py.TypeError("__path__ must be list or None");
         }
-
-        if (attr == null) {
-            // Still looking: maybe it's a Java package?
-            attr = PySystemState.packageManager.lookupName(fullName);
-        }
+        return attr;
+    }
 
+    /**
+     * Add the given attribute to {@code __dict__}, if it is not {@code null} allowing
+     * {@code sys.modules[fullName]} to override.
+     *
+     * @param name of attribute to add
+     * @param fullName by which to check in {@code sys.modules}
+     * @param attr attribute to add (if not overridden)
+     * @return attribute value actually added (may be from {@code sys.modules}) or {@code null}
+     */
+    private PyObject addedSubModule(String name, String fullName, PyObject attr) {
         if (attr != null) {
-            // Allow a package component to change its own meaning
-            PyObject found = modules.__finditem__(fullName);
-            if (found != null) {
-                attr = found;
+            if (fullName != null) {
+                // If a module by the full name exists in sys.modules, that takes precedence.
+                PyObject entry = Py.getSystemState().modules.__finditem__(fullName);
+                if (entry != null) {
+                    attr = entry;
+                }
             }
+            // Enter this as an attribute of this module.
             __dict__.__setitem__(name, attr);
-            return attr;
         }
+        return attr;
+    }
 
+    /**
+     * Construct (and intern) the full name of a possible sub-module of this one, using the
+     * {@code __name__} attribute and a simple sub-module name. Return {@code null} if any of these
+     * requirements is missing.
+     *
+     * @param name simple name of (possible) sub-module
+     * @return interned full name or {@code null}
+     */
+    private String getFullName(String name) {
+        if (__dict__ != null) {
+            PyObject pyName = __dict__.__finditem__("__name__");
+            if (pyName != null && name != null && name.length() > 0) {
+                return (pyName.__str__().toString() + '.' + name).intern();
+            }
+        }
         return null;
     }
 
@@ -146,16 +192,24 @@
      * {@inheritDoc}
      * <p>
      * Overridden in {@code PyModule} so that if the base-class {@code __findattr_ex__} is
-     * unsuccessful, it will to search for the named attribute as a module via
-     * {@link #impAttr(String)}.
+     * unsuccessful, it will to search for the named attribute as a Java sub-package. This is
+     * responsible for the automagical import of Java (but not Python) packages when referred to as
+     * attributes.
      */
     @Override
     public PyObject __findattr_ex__(String name) {
+        // Find the attribute in the dictionary
         PyObject attr = super.__findattr_ex__(name);
-        if (attr != null) {
-            return attr;
+        if (attr == null) {
+            // The attribute may be a Java sub-package to auto-import.
+            String fullName = getFullName(name);
+            if (fullName != null) {
+                attr = PySystemState.packageManager.lookupName(fullName);
+                // Any entry in sys.modules to takes precedence.
+                attr = addedSubModule(name, fullName, attr);
+            }
         }
-        return impAttr(name);
+        return attr;
     }
 
     @Override
diff --git a/src/org/python/core/PyObject.java b/src/org/python/core/PyObject.java
--- a/src/org/python/core/PyObject.java
+++ b/src/org/python/core/PyObject.java
@@ -1028,13 +1028,11 @@
     }
 
     /**
-     * This is a variant of {@link #__findattr__(String)} used by the module import logic to find a
-     * sub-module amongst the attributes of an object representing a package. The default behaviour
-     * is to delegate to {@code __findattr__}, but in particular cases it becomes a hook for
-     * specialised search behaviour.
+     * This is a hook called during the import mechanism when the target module is (or may be)
+     * a sub-module of this object.
      *
-     * @param name the name to lookup in this namespace <b>must be an interned string</b>.
-     * @return the value corresponding to name or null if name is not found
+     * @param name relative to this object <b>must be an interned string</b>.
+     * @return corresponding value (a module or package) or {@code null} if not found
      */
     protected PyObject impAttr(String name) {
         return __findattr__(name);
diff --git a/src/org/python/core/imp.java b/src/org/python/core/imp.java
--- a/src/org/python/core/imp.java
+++ b/src/org/python/core/imp.java
@@ -669,12 +669,12 @@
     }
 
     /**
-     * Try to load a module from {@code sys.meta_path}, as a built-in module, or from either the the
-     * {@code __path__} of the enclosing package or {@code sys.path} if the module is being sought
-     * at the top level.
+     * Try to load a Python module from {@code sys.meta_path}, as a built-in module, or from either
+     * the {@code __path__} of the enclosing package or {@code sys.path} if the module is being
+     * sought at the top level.
      *
      * @param name simple name of the module.
-     * @param moduleName fully-qualified (dotted) name of the module (ending in {@code name}.
+     * @param moduleName fully-qualified (dotted) name of the module (ending in {@code name}).
      * @param path {@code __path__} of the enclosing package (or {@code null} if top level).
      * @return the module if we can load it (or {@code null} if we can't).
      */
@@ -966,12 +966,12 @@
             return null;
         }
 
-        PyObject tmp = dict.__finditem__("__package__"); // XXX: why is this not guaranteed set?
+        PyObject tmp = dict.__finditem__("__package__");
         if (tmp != null && tmp != Py.None) {
             if (!Py.isInstance(tmp, PyString.TYPE)) {
                 throw Py.ValueError("__package__ set to non-string");
             }
-            modname = ((PyString)tmp).getString();
+            modname = ((PyString) tmp).getString();
         } else {
             // __package__ not set, so figure it out and set it.
 
@@ -1015,12 +1015,14 @@
         if (Py.getSystemState().modules.__finditem__(modname) == null) {
             if (orig_level < 1) {
                 if (modname.length() > 0) {
-                    Py.warning(Py.RuntimeWarning, String.format("Parent module '%.200s' not found "
-                            + "while handling absolute import", modname));
+                    Py.warning(Py.RuntimeWarning, String.format(
+                            "Parent module '%.200s' not found " + "while handling absolute import",
+                            modname));
                 }
             } else {
-                throw Py.SystemError(String.format("Parent module '%.200s' not loaded, "
-                        + "cannot perform relative import", modname));
+                throw Py.SystemError(String.format(
+                        "Parent module '%.200s' not loaded, " + "cannot perform relative import",
+                        modname));
             }
         }
         return modname.intern();
@@ -1221,7 +1223,7 @@
      * @param top if true, return the top module in the name, otherwise the last
      * @param modDict the __dict__ of the importing module (used to navigate a relative import)
      * @param fromlist list of names being imported
-     * @param level 0=absolute, n&gt;0=relative levels to go up, -1=try relative then absolute.
+     * @param level 0=absolute, n&gt;0=relative levels to go up - 1, -1=try relative then absolute.
      * @return an imported module (Java or Python)
      */
     private static PyObject import_module_level(String name, boolean top, PyObject modDict,
@@ -1266,7 +1268,7 @@
         PyObject topMod = import_next(pkgMod, parentName, firstName, name, fromlist);
 
         if (topMod == Py.None || topMod == null) {
-            // The first attempt failed. This time
+            // The first attempt failed.
             parentName = new StringBuilder("");
             // could throw ImportError
             if (level > 0) {
@@ -1328,14 +1330,13 @@
 
     /**
      * Ensure that the items mentioned in the from-list of an import are actually present, even if
-     * they are modules we have not imported yet.     *
+     * they are modules we have not imported yet.
      *
      * @param mod module we are importing from
      * @param fromlist tuple of names to import
      * @param name of module we are importing from (as given, may be relative)
-     * @param recursive if true, when the from-list includes "*", do not import __all__
+     * @param recursive true, when this method calls itself
      */
-    // XXX: effect of the recursive argument is hard to fathom.
     private static void ensureFromList(PyObject mod, PyObject fromlist, String name,
             boolean recursive) {
         // This can happen with imports like "from . import foo"

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sat Mar  9 02:06:02 2019
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 09 Mar 2019 07:06:02 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Update_NEWS_with_recently_?=
 =?utf-8?q?closed_bugs=2E?=
Message-ID: <20190309070602.1.18E30837B15E9150@mg.python.org>

https://hg.python.org/jython/rev/a3a9d32659c0
changeset:   8222:a3a9d32659c0
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Fri Mar 08 23:32:51 2019 +0000
summary:
  Update NEWS with recently closed bugs.

Also remove a spurious import added (I think) by PyDev.

files:
  Lib/test/test_import_jy.py |  1 -
  NEWS                       |  3 +++
  2 files changed, 3 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_import_jy.py b/Lib/test/test_import_jy.py
--- a/Lib/test/test_import_jy.py
+++ b/Lib/test/test_import_jy.py
@@ -14,7 +14,6 @@
 import zipfile
 from test import test_support
 from test_chdir import read, safe_mktemp, COMPILED_SUFFIX
-from doctest import script_from_examples
 
 class MislabeledImportTestCase(unittest.TestCase):
 
diff --git a/NEWS b/NEWS
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,9 @@
 
 Development tip
   Bugs fixed
+    - [ 2654 ] Imported modules allow access to submodules
+    - [ 2362 ] Import * doesn't work on JDK9 for java.*, jdk.* namespaces
+    - [ 2630 ] Cannot import java.awt.* on Java 9
     - [ 2663 ] Remove dependency on javax.xml.bind.DatatypeConverter
     - [ 2726 ] os.uname() throws IllegalArgumentException on Windows (Chinese localisation)
     - [ 2719 ] Divergence of __str__ and __repr__ from CPython

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sat Mar 16 12:15:41 2019
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 16 Mar 2019 16:15:41 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Further_comments_in_imp=2E?=
 =?utf-8?q?java_and_clean-up_for_intelligibility=2E?=
Message-ID: <20190316161541.1.F56193A84AFF91B4@mg.python.org>

https://hg.python.org/jython/rev/70493dc7d396
changeset:   8223:70493dc7d396
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Sat Mar 16 13:08:08 2019 +0000
summary:
  Further comments in imp.java and clean-up for intelligibility.

Re-works logic of imp.loadFromSource, hopefully now clearer and without some
needless File objects.

files:
  Lib/test/test_import.py                                 |   10 +-
  src/org/python/core/JavaImportHelper.java               |    2 -
  src/org/python/core/imp.java                            |  211 ++++++---
  src/org/python/core/packagecache/SysPackageManager.java |    9 +-
  4 files changed, 152 insertions(+), 80 deletions(-)


diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py
--- a/Lib/test/test_import.py
+++ b/Lib/test/test_import.py
@@ -653,8 +653,14 @@
         sys.path[:] = self.orig_sys_path
 
 def test_main(verbose=None):
-    run_unittest(ImportTests, PycRewritingTests, PathsTests,
-        RelativeImportTests, TestSymbolicallyLinkedPackage)
+    run_unittest(
+            ImportTests,
+            PycRewritingTests,
+            PathsTests,
+            RelativeImportTests,
+            TestSymbolicallyLinkedPackage
+        )
+
 
 if __name__ == '__main__':
     # Test needs to be a package, so we can do relative imports.
diff --git a/src/org/python/core/JavaImportHelper.java b/src/org/python/core/JavaImportHelper.java
--- a/src/org/python/core/JavaImportHelper.java
+++ b/src/org/python/core/JavaImportHelper.java
@@ -41,7 +41,6 @@
             for (String fromName : stringFromlist) {
                 if (isJavaClass(packageName, fromName)) {
                     packageAdded = addPackage(packageName, packageAdded);
-
                 }
             }
 
@@ -218,5 +217,4 @@
         }
         return packageAdded;
     }
-
 }
diff --git a/src/org/python/core/imp.java b/src/org/python/core/imp.java
--- a/src/org/python/core/imp.java
+++ b/src/org/python/core/imp.java
@@ -406,21 +406,47 @@
         return new CodeData(classFileData, sourceLastModified, ar.getFilename());
     }
 
-    public static byte[] compileSource(String name, File file) {
-        return compileSource(name, file, null);
-    }
-
-    public static byte[] compileSource(String name, File file, String sourceFilename) {
-        return compileSource(name, file, sourceFilename, null);
+    /**
+     * Compile Python source in file to a class file represented by a byte array.
+     *
+     * @param name of module (class name will be name$py)
+     * @param source file containing the source
+     * @return Java byte code as array
+     */
+    public static byte[] compileSource(String name, File source) {
+        return compileSource(name, source, null);
     }
 
-    public static byte[] compileSource(String name, File file, String sourceFilename,
+    /**
+     * Compile Python source in file to a class file represented by a byte array.
+     *
+     * @param name of module (class name will be name$py)
+     * @param source file containing the source
+     * @param filename explicit source file name (or {@code null} to use that in source)
+     * @return Java byte code as array
+     */
+    public static byte[] compileSource(String name, File source, String filename) {
+        if (filename == null) {
+            filename = source.toString();
+        }
+        long mtime = source.lastModified();
+        return compileSource(name, makeStream(source), filename, mtime);
+    }
+
+    /**
+     * Compile Python source in file to a class file represented by a byte array.
+     *
+     * @param name of module (class name will be name$py)
+     * @param source file containing the source
+     * @param sourceFilename explicit source file name (or {@code null} to use that in source)
+     * @param compiledFilename ignored (huh?)
+     * @return Java byte code as array
+     * @deprecated Use {@link #compileSource(String, File, String, String)} instead.
+     */
+    @Deprecated
+    public static byte[] compileSource(String name, File source, String sourceFilename,
             String compiledFilename) {
-        if (sourceFilename == null) {
-            sourceFilename = file.toString();
-        }
-        long mtime = file.lastModified();
-        return compileSource(name, makeStream(file), sourceFilename, mtime);
+        return compileSource(name, source, sourceFilename);
     }
 
     /** Remove the last three characters of a file name and add the compiled suffix "$py.class". */
@@ -478,11 +504,29 @@
         }
     }
 
-    public static byte[] compileSource(String name, InputStream fp, String filename) {
-        return compileSource(name, fp, filename, NO_MTIME);
+    /**
+     * Compile Python source to a class file represented by a byte array.
+     *
+     * @param name of module (class name will be name$py)
+     * @param source open input stream (will be closed)
+     * @param filename of source (or {@code null} if unknown)
+     * @return Java byte code as array
+     */
+    public static byte[] compileSource(String name, InputStream source, String filename) {
+        return compileSource(name, source, filename, NO_MTIME);
     }
 
-    public static byte[] compileSource(String name, InputStream fp, String filename, long mtime) {
+    /**
+     * Compile Python source to a class file represented by a byte array.
+     *
+     * @param name of module (class name will be name$py)
+     * @param source open input stream (will be closed)
+     * @param filename of source (or {@code null} if unknown)
+     * @param mtime last-modified time of source, to annotate class
+     * @return Java byte code as array
+     */
+    public static byte[] compileSource(String name, InputStream source, String filename,
+            long mtime) {
         ByteArrayOutputStream ofp = new ByteArrayOutputStream();
         try {
             if (filename == null) {
@@ -490,10 +534,12 @@
             }
             org.python.antlr.base.mod node;
             try {
-                node = ParserFacade.parse(fp, CompileMode.exec, filename, new CompilerFlags());
+                // Compile source to AST
+                node = ParserFacade.parse(source, CompileMode.exec, filename, new CompilerFlags());
             } finally {
-                fp.close();
+                source.close();
             }
+            // Generate code
             Module.compile(node, ofp, name + "$py", filename, true, false, null, mtime);
             return ofp.toByteArray();
         } catch (Throwable t) {
@@ -790,61 +836,86 @@
      */
     static PyObject loadFromSource(PySystemState sys, String name, String modName,
             String location) {
-        String dirName = sys.getPath(location);
-        String sourceName = "__init__.py";
-        String compiledName = makeCompiledFilename(sourceName);
-        // display names are for identification purposes (e.g. __file__): when entry is
-        // null it forces java.io.File to be a relative path (e.g. foo/bar.py instead of
-        // /tmp/foo/bar.py)
-        String displayDirName = location.equals("") ? null : location;
-        String displaySourceName = new File(new File(displayDirName, name), sourceName).getPath();
-        String displayCompiledName =
-                new File(new File(displayDirName, name), compiledName).getPath();
+        File sourceFile;     // location/name/__init__.py or location/name.py
+        File compiledFile;   // location/name/__init__$py.class or location/name$py.class
+        boolean haveSource = false, haveCompiled = false;
+
+        // display* names are for mainly identification purposes (e.g. __file__)
+        String displayLocation = (location.equals("") || location.equals(",")) ? null : location;
+        String displaySourceName, displayCompiledName;
+
+        try {
+            /*
+             * Distinguish package and module cases by choosing File objects sourceFile and
+             * compiledFile based on name/__init__ or name. haveSource and haveCompiled are set true
+             * if the corresponding source or compiled files exist, and this is what steers the
+             * loading in the second part of the process.
+             */
+            String dirName = sys.getPath(location);
+            File dir = new File(dirName, name);
+
+            if (dir.isDirectory()) {
+                // This should be a package: location/name
+                File displayDir = new File(displayLocation, name);
 
-        // Create file objects to check for a Python package
-        File dir = new File(dirName, name);                 // entry/name/
-        File sourceFile = new File(dir, sourceName);        // entry/name/__init__.py
-        File compiledFile = new File(dir, compiledName);    // entry/name/__init__$py.class
+                // Source is location/name/__init__.py
+                String sourceName = "__init__.py";
+                sourceFile = new File(dir, sourceName);
+                displaySourceName = new File(displayDir, sourceName).getPath();
+
+                // Compiled is location/name/__init__$py.class
+                String compiledName = makeCompiledFilename(sourceName);
+                compiledFile = new File(dir, compiledName);
+                displayCompiledName = new File(displayDir, compiledName).getPath();
 
-        boolean isPackage = false;
-        try {
-            if (dir.isDirectory()) {
-                if (caseok(dir, name) && (sourceFile.isFile() || compiledFile.isFile())) {
-                    isPackage = true;
+                // Check the directory name is ok according to case-matching option and platform.
+                if (caseok(dir, name)) {
+                    haveSource = sourceFile.isFile();
+                    haveCompiled = compiledFile.isFile();
+                }
+
+                if (haveSource || haveCompiled) {
+                    // Create a PyModule (uninitialised) for name, called modName in sys.modules
+                    PyModule m = addModule(modName);
+                    PyString filename = Py.fileSystemEncode(displayDir.getPath());
+                    m.__dict__.__setitem__("__path__", new PyList(new PyObject[] {filename}));
                 } else {
+                    /*
+                     * There is neither source nor compiled code for __init__.py. In Jython, this
+                     * message warning is premature, as there may be a Java package by this name.
+                     */
                     String printDirName = PyString.encode_UnicodeEscape(dir.getPath(), '\'');
                     Py.warning(Py.ImportWarning, String.format(
                             "Not importing directory %s: missing __init__.py", printDirName));
-                    return null;
                 }
-            }
-        } catch (SecurityException e) {
-            // ok
-        }
+
+            } else {
+                // This is a (non-package) module: location/name
+
+                // Source is location/name.py
+                String sourceName = name + ".py";
+                sourceFile = new File(dirName, sourceName);     // location/name.py
+                displaySourceName = new File(displayLocation, sourceName).getPath();
+
+                // Compiled is location/name$py.class
+                String compiledName = makeCompiledFilename(sourceName);
+                compiledFile = new File(dirName, compiledName); // location/name$py.class
+                displayCompiledName = new File(displayLocation, compiledName).getPath();
 
-        if (!isPackage) {
-            // The source is entry/name.py and compiled is entry/name$py.class
-            Py.writeDebug(IMPORT_LOG, "trying source " + dir.getPath());
-            sourceName = name + ".py";
-            compiledName = makeCompiledFilename(sourceName);
-            displaySourceName = new File(displayDirName, sourceName).getPath();
-            displayCompiledName = new File(displayDirName, compiledName).getPath();
-            sourceFile = new File(dirName, sourceName);
-            compiledFile = new File(dirName, compiledName);
-        } else {
-            // Create a PyModule (uninitialised) for name.py, called modName in sys.modules
-            PyModule m = addModule(modName);
-            PyObject filename = Py.newStringOrUnicode(new File(displayDirName, name).getPath());
-            // FIXME: FS encode file name for __path__
-            m.__dict__.__setitem__("__path__", new PyList(new PyObject[] {filename}));
-        }
+                // Check file names exist and ok according to case-matching option and platform.
+                haveSource = sourceFile.isFile() && caseok(sourceFile, sourceName);
+                haveCompiled = compiledFile.isFile() && caseok(compiledFile, compiledName);
+            }
 
-        // Load and execute the module, from its compiled or source form.
-        try {
-            // Try to create the module from source or an existing compiled class.
-            if (sourceFile.isFile() && caseok(sourceFile, sourceName)) {
+            /*
+             * Now we are ready to load and execute the module in sourceFile or compiledFile, from
+             * its compiled or source form, as directed by haveSource and haveCompiled.
+             */
+            if (haveSource) {
+                // Try to create the module from source or an existing compiled class.
                 long pyTime = sourceFile.lastModified();
-                if (compiledFile.isFile() && caseok(compiledFile, compiledName)) {
+
+                if (haveCompiled) {
                     // We have the compiled file and will use that if it is not out of date
                     Py.writeDebug(IMPORT_LOG, "trying precompiled " + compiledFile.getPath());
                     long classTime = compiledFile.lastModified();
@@ -858,21 +929,23 @@
                         }
                     }
                 }
-                // The compiled class is not present, was out of date, or trying to use it failed.
+
+                // The compiled class is not present, is out of date, or using it failed somehow.
+                Py.writeDebug(IMPORT_LOG, "trying source " + sourceFile.getPath());
                 return createFromSource(modName, makeStream(sourceFile), displaySourceName,
                         compiledFile.getPath(), pyTime);
-            }
 
-            // If no source, try loading compiled
-            Py.writeDebug(IMPORT_LOG,
-                    "trying precompiled with no source " + compiledFile.getPath());
-            if (compiledFile.isFile() && caseok(compiledFile, compiledName)) {
+            } else if (haveCompiled) {
+                // There is no source, try loading compiled
+                Py.writeDebug(IMPORT_LOG,
+                        "trying precompiled with no source " + compiledFile.getPath());
                 return createFromPyClass(modName, makeStream(compiledFile), //
                         false, // throw ImportError here if this fails
                         displaySourceName, displayCompiledName, NO_MTIME, CodeImport.compiled_only);
             }
+
         } catch (SecurityException e) {
-            // ok
+            // We were prevented from reading some essential file, so pretend we didn't find it.
         }
         return null;
     }
diff --git a/src/org/python/core/packagecache/SysPackageManager.java b/src/org/python/core/packagecache/SysPackageManager.java
--- a/src/org/python/core/packagecache/SysPackageManager.java
+++ b/src/org/python/core/packagecache/SysPackageManager.java
@@ -257,14 +257,9 @@
     public boolean packageExists(String pkg, String name) {
         if (packageExists(this.searchPath, pkg, name)) {
             return true;
-        }
-
-        PySystemState sys = Py.getSystemState();
-        if (sys.getClassLoader() == null && packageExists(sys.path, pkg, name)) {
-            return true;
         } else {
-            return false;
+            PySystemState sys = Py.getSystemState();
+            return sys.getClassLoader() == null && packageExists(sys.path, pkg, name);
         }
     }
-
 }

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sat Mar 16 12:15:42 2019
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 16 Mar 2019 16:15:42 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Support_Unicode_in_zxJDBC_?=
 =?utf-8?q?exceptions=2E?=
Message-ID: <20190316161542.1.7603341EB8DA6029@mg.python.org>

https://hg.python.org/jython/rev/d802aba05df2
changeset:   8224:d802aba05df2
user:        ?? <kuiseng.zhb at alibaba-inc.com>
date:        Sat Mar 16 13:31:02 2019 +0000
summary:
  Support Unicode in zxJDBC exceptions.

files:
  ACKNOWLEDGMENTS                       |  1 +
  NEWS                                  |  1 +
  src/com/ziclix/python/sql/zxJDBC.java |  2 +-
  3 files changed, 3 insertions(+), 1 deletions(-)


diff --git a/ACKNOWLEDGMENTS b/ACKNOWLEDGMENTS
--- a/ACKNOWLEDGMENTS
+++ b/ACKNOWLEDGMENTS
@@ -182,6 +182,7 @@
     Tom Bech
     Richie Bendall
     Raymond Ferguson
+    yishenggudou (??)
 
 Local Variables:
 mode: indented-text
diff --git a/NEWS b/NEWS
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,7 @@
 
 Development tip
   Bugs fixed
+    - [ GH-131 ] Support Unicode in zxJDBC exceptions.
     - [ 2654 ] Imported modules allow access to submodules
     - [ 2362 ] Import * doesn't work on JDK9 for java.*, jdk.* namespaces
     - [ 2630 ] Cannot import java.awt.* on Java 9
diff --git a/src/com/ziclix/python/sql/zxJDBC.java b/src/com/ziclix/python/sql/zxJDBC.java
--- a/src/com/ziclix/python/sql/zxJDBC.java
+++ b/src/com/ziclix/python/sql/zxJDBC.java
@@ -327,7 +327,7 @@
      * @return PyException
      */
     public static PyException makeException(PyObject type, String msg) {
-        return Py.makeException(type, msg == null ? Py.EmptyString : Py.newString(msg));
+        return Py.makeException(type, msg == null ? Py.EmptyString : Py.newStringOrUnicode(msg));
     }
 
     /**

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Mon Mar 18 02:21:14 2019
From: jython-checkins at python.org (jeff.allen)
Date: Mon, 18 Mar 2019 06:21:14 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Revise_error_messages_from?=
 =?utf-8?q?_oversize_byte_code_work-around=2E_=28Fixes_=232732=2E=29?=
Message-ID: <20190318062114.1.E130DA7163D4C8C5@mg.python.org>

https://hg.python.org/jython/rev/96917c0883da
changeset:   8225:96917c0883da
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Sun Mar 17 08:52:26 2019 +0000
summary:
  Revise error messages from oversize byte code work-around. (Fixes #2732.)

Previous output gave advice that no longer works. This change is primarily to
update those messages. In the process, code is rationalised slightly to compute
the messages more lazily.

The property for specifying CPython to Jython, formerly "cpython_cmd" is
namespaced as other properties we define to "python.cpython2".

files:
  NEWS                                |    1 +
  src/org/python/compiler/Module.java |  129 +++++++--------
  2 files changed, 64 insertions(+), 66 deletions(-)


diff --git a/NEWS b/NEWS
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,7 @@
 
 Development tip
   Bugs fixed
+    - [ 2732 ] Regression in large module support for pip
     - [ GH-131 ] Support Unicode in zxJDBC exceptions.
     - [ 2654 ] Imported modules allow access to submodules
     - [ 2362 ] Import * doesn't work on JDK9 for java.*, jdk.* namespaces
diff --git a/src/org/python/compiler/Module.java b/src/org/python/compiler/Module.java
--- a/src/org/python/compiler/Module.java
+++ b/src/org/python/compiler/Module.java
@@ -717,53 +717,57 @@
         module.mainCode = main;
     }
 
+    /** Property naming the cpython executable. */
+    private static String CPYTHON = "python.cpython2";
+
+    // Error message formats required by loadPyBytecode
+    private static String TRIED_CREATE_PYC_MSG =
+            "\nJython tried to create a pyc-file by executing\n    %s\nwhich failed because %s";
+    private static String LARGE_METHOD_MSG = "Module or method too large in `%s`.";
+    private static String PLEASE_PROVIDE_MSG =
+            "\n\nPlease provide a CPython 2.7 bytecode file (.pyc), e.g. run"
+                    + "\n    python -m py_compile %s";
+    private static String CPYTHON_CMD_MSG =
+            "\n\nAlternatively, specify a CPython 2.7 command via the " //
+                    + CPYTHON + " property, e.g.:" //
+                    + "\n    jython -D" + CPYTHON + "=python" //
+                    + "\nor (e.g. for pip) through the environment variable JYTHON_OPTS:" //
+                    + "\n    export JYTHON_OPTS=\"-D" + CPYTHON + "=python\"\n";
+
     private static PyBytecode loadPyBytecode(String filename, boolean try_cpython)
-            throws RuntimeException
-    {
+            throws RuntimeException {
         if (filename.startsWith(ClasspathPyImporter.PYCLASSPATH_PREFIX)) {
             ClassLoader cld = Py.getSystemState().getClassLoader();
             if (cld == null) {
                 cld = imp.getParentClassLoader();
             }
-            URL py_url = cld.getResource(filename.replace(
-                    ClasspathPyImporter.PYCLASSPATH_PREFIX, ""));
+            URL py_url =
+                    cld.getResource(filename.replace(ClasspathPyImporter.PYCLASSPATH_PREFIX, ""));
             if (py_url != null) {
                 filename = py_url.getPath();
             } else {
                 // Should never happen, but let's play it safe and treat this case.
-                throw new RuntimeException(
-                        "\nEncountered too large method code in \n"+filename+",\n"+
-                        "but couldn't resolve that filename within classpath.\n"+
-                        "Make sure the source file is at a proper location.");
+                throw new RuntimeException(String.format(LARGE_METHOD_MSG, filename)
+                        + "but couldn't resolve that filename within classpath.\n"
+                        + "Make sure the source file is at a proper location.");
             }
         }
-        String cpython_cmd_msg =
-                "\n\nAlternatively provide proper CPython 2.7 execute command via"+
-                "\ncpython_cmd property, e.g. call "+
-                "\n    jython -J-Dcpython_cmd=python"+
-                "\nor if running pip on Jython:"+
-                "\n    pip install --global-option=\"-J-Dcpython_cmd=python\" <package>";
-        String large_method_msg = "\nEncountered too large method code in \n"+filename+"\n";
-        String please_provide_msg =
-                "\nPlease provide a CPython 2.7 bytecode file (.pyc) to proceed, e.g. run"+
-                "\npython -m py_compile "+filename+"\nand try again.";
 
-        String pyc_filename = filename+"c";
+        String pyc_filename = filename + "c";
         File pyc_file = new File(pyc_filename);
         if (pyc_file.exists()) {
             PyFile f = new PyFile(pyc_filename, "rb", 4096);
             byte[] bts = f.read(8).toBytes();
-            int magic = (bts[1]<< 8) & 0x0000FF00 |
-                        (bts[0]<< 0) & 0x000000FF;
-//            int mtime_pyc = (bts[7]<<24) & 0xFF000000 |
-//                            (bts[6]<<16) & 0x00FF0000 |
-//                            (bts[5]<< 8) & 0x0000FF00 |
-//                            (bts[4]<< 0) & 0x000000FF;
+            int magic = (bts[1] << 8) & 0x0000FF00 | (bts[0] << 0) & 0x000000FF;
+            // int mtime_pyc = (bts[7]<<24) & 0xFF000000 |
+            // (bts[6]<<16) & 0x00FF0000 |
+            // (bts[5]<< 8) & 0x0000FF00 |
+            // (bts[4]<< 0) & 0x000000FF;
             if (magic != 62211) { // check Python 2.7 bytecode
-                throw new RuntimeException(large_method_msg+
-                        "\n"+pyc_filename+
-                        "\ncontains wrong bytecode version, not CPython 2.7 bytecode."+
-                        please_provide_msg);
+                throw new RuntimeException(
+                        String.format(LARGE_METHOD_MSG, filename) //
+                                + "\n'" + pyc_filename + "' is not CPython 2.7 bytecode." //
+                                + String.format(PLEASE_PROVIDE_MSG, filename));
             }
             _marshal.Unmarshaller un = new _marshal.Unmarshaller(f);
             PyObject code = un.load();
@@ -771,24 +775,24 @@
             if (code instanceof PyBytecode) {
                 return (PyBytecode) code;
             }
-            throw new RuntimeException(large_method_msg+
-                    "\n"+pyc_filename+
-                    "\ncontains invalid bytecode."+
-                    please_provide_msg);
+            throw new RuntimeException(String.format(LARGE_METHOD_MSG, filename) //
+                    + "\n'" + pyc_filename + "' contains invalid bytecode."
+                    + String.format(PLEASE_PROVIDE_MSG, filename));
+
         } else {
-            String CPython_command = System.getProperty("cpython_cmd");
+            String CPython_command = System.getProperty(CPYTHON);
             if (try_cpython && CPython_command != null) {
                 // check version...
-                String command_ver = CPython_command+" --version";
-                String command = CPython_command+" -m py_compile "+filename;
-                String tried_create_pyc_msg = "\nTried to create pyc-file by executing\n"+
-                        command+"\nThis failed because of\n";
+                String command_ver = CPython_command + " --version";
+                String command = CPython_command + " -m py_compile " + filename;
                 Exception exc = null;
                 int result = 0;
+                String reason;
                 try {
                     Process p = Runtime.getRuntime().exec(command_ver);
                     // Python 2.7 writes version to error-stream for some reason:
-                    BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream()));
+                    BufferedReader br =
+                            new BufferedReader(new InputStreamReader(p.getErrorStream()));
                     String cp_version = br.readLine();
                     while (br.readLine() != null) {}
                     br.close();
@@ -801,19 +805,15 @@
                     }
                     result = p.waitFor();
                     if (!cp_version.startsWith("Python 2.7.")) {
-                        throw new RuntimeException(large_method_msg+
-                                tried_create_pyc_msg+
-                                "wrong Python version: "+cp_version+"."+
-                                "\nRequired is Python 2.7.x.\n"+
-                                please_provide_msg+cpython_cmd_msg);
+                        reason = cp_version + " has been provided, but 2.7.x is required.";
+                        throw new RuntimeException(String.format(LARGE_METHOD_MSG, filename)
+                                + String.format(TRIED_CREATE_PYC_MSG, command, reason)
+                                + String.format(PLEASE_PROVIDE_MSG, filename) + CPYTHON_CMD_MSG);
                     }
+                } catch (InterruptedException | IOException e) {
+                    exc = e;
                 }
-                catch (InterruptedException ie) {
-                    exc = ie;
-                }
-                catch (IOException ioe) {
-                    exc = ioe;
-                }
+
                 if (exc == null && result == 0) {
                     try {
                         Process p = Runtime.getRuntime().exec(command);
@@ -821,22 +821,19 @@
                         if (result == 0) {
                             return loadPyBytecode(filename, false);
                         }
-                    }
-                    catch (InterruptedException ie) {
-                        exc = ie;
-                    }
-                    catch (IOException ioe) {
-                        exc = ioe;
+                    } catch (InterruptedException | IOException e) {
+                        exc = e;
                     }
                 }
-                String exc_msg = large_method_msg+
-                        tried_create_pyc_msg+
-                        (exc != null ? exc.toString() : "bad return: "+result)+".\n"+
-                        please_provide_msg+cpython_cmd_msg;
-                throw exc != null ? new RuntimeException(exc_msg, exc) : new RuntimeException(exc_msg);
+                reason = exc != null ? "of " + exc.toString() : "of a bad return: " + result;
+                String exc_msg = String.format(LARGE_METHOD_MSG, filename)
+                        + String.format(TRIED_CREATE_PYC_MSG, command, reason)
+                        + String.format(PLEASE_PROVIDE_MSG, filename) + CPYTHON_CMD_MSG;
+                throw exc != null ? new RuntimeException(exc_msg, exc)
+                        : new RuntimeException(exc_msg);
             } else {
-                throw new RuntimeException(large_method_msg+
-                        please_provide_msg+cpython_cmd_msg);
+                throw new RuntimeException(String.format(LARGE_METHOD_MSG, filename)
+                        + String.format(PLEASE_PROVIDE_MSG, filename) + CPYTHON_CMD_MSG);
             }
         }
     }
@@ -875,7 +872,7 @@
      * Implement a simplified base64 encoding compatible with the decoding in BytecodeLoader. This
      * encoder adds no '=' padding or line-breaks. equivalent to
      * {@code binascii.b2a_base64(bytes).rstrip('=\n')}.
-     * 
+     *
      * @param data to encode
      * @return the string encoding the data
      */
@@ -947,7 +944,7 @@
      * Note that this approach is provisional. In future, Jython might contain the bytecode directly
      * as bytecode-objects. The current approach was feasible with far less complicated JVM
      * bytecode-manipulation, but needs special treatment after class-loading.
-     * 
+     *
      * @param name of the method or function being generated
      * @param code_str Base64 encoded CPython byte code
      * @param module currently being defined as a class file
@@ -979,7 +976,7 @@
 
     /**
      * Create and write a Python module as a Java class file.
-     * 
+     *
      * @param node AST of the module to write
      * @param ostream stream onto which to write it
      * @param name

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Mon Mar 18 02:21:14 2019
From: jython-checkins at python.org (jeff.allen)
Date: Mon, 18 Mar 2019 06:21:14 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Improve_complex_MRO_handli?=
 =?utf-8?q?ng=2E_=28Fixes_bjo_=232445=2E=29?=
Message-ID: <20190318062114.1.F3CAE7FC9A0A8920@mg.python.org>

https://hg.python.org/jython/rev/3c4f2c14b808
changeset:   8226:3c4f2c14b808
user:        Jim Baker <jim.baker at python.org>
date:        Sun Mar 17 20:57:18 2019 +0000
summary:
  Improve complex MRO handling. (Fixes bjo #2445.)

This change fixes a defect in MRO processing (as posted by Jim). It also adds a
test class (adapted by Jeff) with complex inheritance between its inner classes,
and a regression test that would fail before Jim's patch was applied.

files:
  Lib/test/test_java_integration.py               |  25 +++-
  NEWS                                            |   1 +
  src/org/python/core/PyJavaType.java             |  25 +++-
  tests/java/org/python/tests/mro/TortureMRO.java |  51 ++++++++++
  4 files changed, 89 insertions(+), 13 deletions(-)


diff --git a/Lib/test/test_java_integration.py b/Lib/test/test_java_integration.py
--- a/Lib/test/test_java_integration.py
+++ b/Lib/test/test_java_integration.py
@@ -234,6 +234,7 @@
 Keywords.with = lambda self: "with"
 Keywords.yield = lambda self: "yield"
 
+
 class PyReservedNamesTest(unittest.TestCase):
     "Access to reserved words"
 
@@ -330,6 +331,7 @@
     def test_yield(self):
         self.assertEquals(self.kws.yield(), "yield")
 
+
 class ImportTest(unittest.TestCase):
     def test_bad_input_exception(self):
         self.assertRaises(ValueError, __import__, '')
@@ -378,6 +380,13 @@
         x = String('test')
         self.assertRaises(TypeError, list, x)
 
+    def test_null_tostring(self):
+        # http://bugs.jython.org/issue1819
+        nts = NullToString()
+        self.assertEqual(repr(nts), '')
+        self.assertEqual(str(nts), '')
+        self.assertEqual(unicode(nts), '')
+
 class JavaDelegationTest(unittest.TestCase):
     def test_list_delegation(self):
         for c in ArrayList, Vector:
@@ -531,6 +540,9 @@
         self.assertEquals(7, m.initial)
         self.assertEquals(None, m.nonexistent, "Nonexistent fields should be passed on to the Map")
 
+
+class JavaMROTest(unittest.TestCase):
+
     def test_adding_on_interface(self):
         GetitemAdder.addPredefined()
         class UsesInterfaceMethod(FirstPredefinedGetitem):
@@ -543,13 +555,6 @@
         self.assertRaises(TypeError, __import__, "org.python.tests.mro.ConfusedOnImport")
         self.assertRaises(TypeError, GetitemAdder.addPostdefined)
 
-    def test_null_tostring(self):
-        # http://bugs.jython.org/issue1819
-        nts = NullToString()
-        self.assertEqual(repr(nts), '')
-        self.assertEqual(str(nts), '')
-        self.assertEqual(unicode(nts), '')
-
     def test_diamond_inheritance_of_iterable_and_map(self):
         """Test deeply nested diamond inheritance of Iterable and Map, as see in some Clojure classes"""
         # http://bugs.jython.org/issue1878
@@ -570,6 +575,11 @@
         self.assertEqual(set(m), set(["abc", "xyz"]))
         self.assertEqual(m["abc"], 42)
 
+    def test_diamond_lattice_inheritance(self):
+        # http://bugs.jython.org/issue2445
+        from org.python.tests.mro import TortureMRO
+
+
 def roundtrip_serialization(obj):
     """Returns a deep copy of an object, via serializing it
  
@@ -946,6 +956,7 @@
         JavaDelegationTest,
         JavaReservedNamesTest,
         JavaStringTest,
+        JavaMROTest,
         JavaWrapperCustomizationTest,
         PyReservedNamesTest,
         SecurityManagerTest,
diff --git a/NEWS b/NEWS
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,7 @@
 
 Development tip
   Bugs fixed
+    - [ 2445 ] Eclipse's DelegatingFeatureMap has MRO conflict
     - [ 2732 ] Regression in large module support for pip
     - [ GH-131 ] Support Unicode in zxJDBC exceptions.
     - [ 2654 ] Imported modules allow access to submodules
diff --git a/src/org/python/core/PyJavaType.java b/src/org/python/core/PyJavaType.java
--- a/src/org/python/core/PyJavaType.java
+++ b/src/org/python/core/PyJavaType.java
@@ -257,7 +257,17 @@
                     PyList types = new PyList();
                     Set<Class<?>> proxySet = Generic.set();
                     for (PyJavaType othertype : conflictedAttributes) {
-                        if (othertype.modified != null && othertype.modified.contains(method)) {
+                        /*
+                         * Ignore any pairings of types that are in a superclass/superinterface
+                         * relationship with each other. This problem is a false positive that
+                         * happens because of the automatic addition of methods so that Java classes
+                         * behave more like their corresponding Python types, such as adding sort or
+                         * remove. See http://bugs.jython.org/issue2445
+                         */
+                        if (othertype.modified != null && othertype.modified.contains(method)
+                                && !(othertype.getProxyType().isAssignableFrom(type.getProxyType())
+                                        || type.getProxyType()
+                                                .isAssignableFrom(othertype.getProxyType()))) {
                             types.add(othertype);
                             proxySet.add(othertype.getProxyType());
                         }
@@ -269,13 +279,16 @@
                      * __iter__. Annoying but necessary logic. See http://bugs.jython.org/issue1878
                      */
                     if (method.equals("__iter__")
-                            && proxySet.equals(Generic.set(Iterable.class, Map.class))) {
+                            && Generic.set(Iterable.class, Map.class).containsAll(proxySet)) {
                         continue;
                     }
-                    throw Py.TypeError(String.format(
-                            "Supertypes that share a modified attribute "
-                                    + "have an MRO conflict[attribute=%s, supertypes=%s, type=%s]",
-                            method, types, this.getName()));
+
+                    if (types.size() > 0) {
+                        throw Py.TypeError(String.format(
+                                "Supertypes that share a modified attribute "
+                                        + "have an MRO conflict[attribute=%s, supertypes=%s, type=%s]",
+                                method, types, this.getName()));
+                    }
                 }
             }
         }
diff --git a/tests/java/org/python/tests/mro/TortureMRO.java b/tests/java/org/python/tests/mro/TortureMRO.java
new file mode 100644
--- /dev/null
+++ b/tests/java/org/python/tests/mro/TortureMRO.java
@@ -0,0 +1,51 @@
+// Copyright (c) Jython Developers.
+// Licensed to the Python Software Foundation under a Contributor Agreement.
+
+package org.python.tests.mro;
+
+/**
+ * A class providing interface and abstract class relationships that approximate the structure of
+ * org.eclipse.emf.ecore.util.DelegatingFeatureMap, in order to exercise b.j.o issue 2445. The
+ * complex inheritance (more complex than diamond inheritance) confused PyJavaType handling of the
+ * MRO. This class is imported by
+ * {@code test_java_integration.JavaMROTest.test_diamond_lattice_inheritance} as a test.
+ * <p>
+ * An invocation at the prompt (for debugging use), and output before the fix, is:
+ *
+ * <pre>
+ * PS &gt; dist\bin\jython -S -c"from org.python.tests.mro import TortureMRO"
+ * Traceback (most recent call last):
+ *   File "&lt;string&gt;", line 1, in &lt;module&gt;
+ * TypeError: Supertypes that share a modified attribute have an MRO
+ * conflict[attribute=sort, supertypes=[&lt;type 'java.util.List'&gt;,
+ * &lt;type 'java.util.AbstractList'&gt;], type=TortureMRO$Target]
+ * </pre>
+ */
+public class TortureMRO {
+
+    interface Entry {} // Was FeatureMap.Entry
+
+    interface Thing {} // Was EStructuralFeature.Setting
+
+    interface EList<E> extends java.util.List<E> {}
+
+    interface IEList<E> extends EList<E> {}
+
+    interface FList extends EList<Entry> {} // Was FeatureMap
+
+    interface FIEListThing extends FList, IEList<Entry>, Thing {}  // Was 2 FeatureMap.Internal
+
+    abstract static class AbstractEList<E> extends java.util.AbstractList<E> implements EList<E> {}
+
+    abstract static class DEList<E> extends AbstractEList<E> {}
+
+    interface NList<E> extends EList<E> {}
+
+    abstract static class DNListImpl<E> extends DEList<E> implements NList<E> {}
+
+    abstract static class DNIEListImpl<E> extends DNListImpl<E> implements IEList<E> {}
+
+    abstract static class DNIEListThing<E> extends DNIEListImpl<E> implements Thing {}
+
+    public abstract static class Target extends DNIEListThing<Entry> implements FIEListThing {}
+}

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sat Mar 23 04:50:15 2019
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 23 Mar 2019 08:50:15 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_marshal_to_raise_error_whe?=
 =?utf-8?q?n_fed_unmarshalable_object=2E_Fixes_=232077=2E?=
Message-ID: <20190323085015.1.2A8C7D3641DBB014@mg.python.org>

https://hg.python.org/jython/rev/f5dda19d78c9
changeset:   8228:f5dda19d78c9
user:        Andrew Kuchling <amk at amk.ca>
date:        Fri Mar 22 07:50:54 2019 +0000
summary:
  marshal to raise error when fed unmarshalable object. Fixes #2077.

files:
  ACKNOWLEDGMENTS                      |  1 +
  NEWS                                 |  1 +
  src/org/python/modules/_marshal.java |  4 +---
  3 files changed, 3 insertions(+), 3 deletions(-)


diff --git a/ACKNOWLEDGMENTS b/ACKNOWLEDGMENTS
--- a/ACKNOWLEDGMENTS
+++ b/ACKNOWLEDGMENTS
@@ -183,6 +183,7 @@
     Richie Bendall
     Raymond Ferguson
     yishenggudou (??)
+    Andrew Kuchling
 
 Local Variables:
 mode: indented-text
diff --git a/NEWS b/NEWS
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,7 @@
 
 Development tip
   Bugs fixed
+    - [ 2077 ] marshal doesn't raise error when fed unmarshalable object
     - [ 2445 ] Eclipse's DelegatingFeatureMap has MRO conflict
     - [ 2732 ] Regression in large module support for pip
     - [ GH-131 ] Support Unicode in zxJDBC exceptions.
diff --git a/src/org/python/modules/_marshal.java b/src/org/python/modules/_marshal.java
--- a/src/org/python/modules/_marshal.java
+++ b/src/org/python/modules/_marshal.java
@@ -46,7 +46,6 @@
     private final static char TYPE_DICT = '{';
     private final static char TYPE_CODE = 'c';
     private final static char TYPE_UNICODE = 'u';
-    private final static char TYPE_UNKNOWN = '?';
     private final static char TYPE_SET = '<';
     private final static char TYPE_FROZENSET = '>';
     private final static int MAX_MARSHAL_STACK_DEPTH = 2000;
@@ -239,9 +238,8 @@
                 write_int(code.co_firstlineno);
                 write_object(Py.newString(new String(code.co_lnotab)), depth + 1);
             } else {
-                write_byte(TYPE_UNKNOWN);
+                throw Py.ValueError("unmarshallable object");
             }
-
             depth--;
         }
 

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sat Mar 23 04:50:15 2019
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 23 Mar 2019 08:50:15 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Improvements_to_marshal_an?=
 =?utf-8?q?d_its_error_messages=2C_and_fix_for_bjo_=232744=2E?=
Message-ID: <20190323085015.1.E708069021E8A990@mg.python.org>

https://hg.python.org/jython/rev/1e7671d4af8d
changeset:   8229:1e7671d4af8d
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Sat Mar 23 08:12:27 2019 +0000
summary:
  Improvements to marshal and its error messages, and fix for bjo #2744.

Updating test_marshal, in an attempt to find a failing test for #2077, revealed
other divergences unrelated to #2077. This change addresses those, notably
support for objects with the buffer protocol (#2744). Messages in ValueError
exceptions are improved to be like those from CPython.

files:
  Lib/test/test_marshal.py             |    3 -
  NEWS                                 |    1 +
  src/org/python/modules/_marshal.java |  249 ++++++++------
  3 files changed, 140 insertions(+), 113 deletions(-)


diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
--- a/Lib/test/test_marshal.py
+++ b/Lib/test/test_marshal.py
@@ -136,7 +136,6 @@
             self.assertEqual(type(s), type(new))
         os.unlink(test_support.TESTFN)
 
-    @unittest.skipIf(test_support.is_jython, "FIXME: bjo #2744 buffer not supported")
     def test_buffer(self):
         for s in ["", "Andr? Previn", "abc", " "*10000]:
             with test_support.check_py3k_warnings(("buffer.. not supported",
@@ -252,7 +251,6 @@
         last.append([0])
         self.assertRaises(ValueError, marshal.dumps, head)
 
-    @unittest.skipIf(test_support.is_jython, "FIXME: bjo #2077 ValueError not raised")
     def test_exact_type_match(self):
         # Former bug:
         #   >>> class Int(int): pass
@@ -271,7 +269,6 @@
         testString = 'abc' * size
         marshal.dumps(testString)
 
-    @unittest.skipIf(test_support.is_jython, "FIXME: bjo #2077 ValueError not raised")
     def test_invalid_longs(self):
         # Issue #7019: marshal.loads shouldn't produce unnormalized PyLongs
         invalid_string = 'l\x02\x00\x00\x00\x00\x00\x00\x00'
diff --git a/NEWS b/NEWS
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,7 @@
 
 Development tip
   Bugs fixed
+    - [ 2744 ] Support buffer type in marshal.dump(s)
     - [ 2077 ] marshal doesn't raise error when fed unmarshalable object
     - [ 2445 ] Eclipse's DelegatingFeatureMap has MRO conflict
     - [ 2732 ] Regression in large module support for pip
diff --git a/src/org/python/modules/_marshal.java b/src/org/python/modules/_marshal.java
--- a/src/org/python/modules/_marshal.java
+++ b/src/org/python/modules/_marshal.java
@@ -1,25 +1,31 @@
 package org.python.modules;
 
-import java.math.BigInteger;
 import org.python.core.BaseSet;
+import org.python.core.BufferProtocol;
 import org.python.core.ClassDictInit;
-import org.python.core.PyObject;
-import org.python.core.PyString;
 import org.python.core.Py;
+import org.python.core.PyBUF;
+import org.python.core.PyBuffer;
 import org.python.core.PyBytecode;
 import org.python.core.PyComplex;
 import org.python.core.PyDictionary;
+import org.python.core.PyException;
 import org.python.core.PyFloat;
 import org.python.core.PyFrozenSet;
 import org.python.core.PyInteger;
 import org.python.core.PyList;
 import org.python.core.PyLong;
+import org.python.core.PyObject;
 import org.python.core.PySet;
+import org.python.core.PyString;
 import org.python.core.PyTuple;
+import org.python.core.PyType;
 import org.python.core.PyUnicode;
 import org.python.core.Traverseproc;
 import org.python.core.Visitproc;
 
+import java.math.BigInteger;
+
 public class _marshal implements ClassDictInit {
 
     public static void classDictInit(PyObject dict) {
@@ -112,13 +118,12 @@
 
         // writes output in 15 bit "digits"
         private void write_long(BigInteger x) {
-            int sign = x.signum();
-            if (sign < 0) {
+            boolean negative = x.signum() < 0;
+            if (negative) {
                 x = x.negate();
             }
-            int num_bits = x.bitLength();
-            int num_digits = num_bits / 15 + (num_bits % 15 == 0 ? 0 : 1);
-            write_int(sign < 0 ? -num_digits : num_digits);
+            int num_digits = (x.bitLength() + 14) / 15;
+            write_int(negative ? -num_digits : num_digits);
             BigInteger mask = BigInteger.valueOf(0x7FFF);
             for (int i = 0; i < num_digits; i++) {
                 write_short(x.and(mask).shortValue());
@@ -149,96 +154,108 @@
                 write_byte(TYPE_FALSE);
             } else if (v == Py.True) {
                 write_byte(TYPE_TRUE);
-            } else if (v instanceof PyInteger) {
-                write_byte(TYPE_INT);
-                write_int(((PyInteger) v).asInt());
-            } else if (v instanceof PyLong) {
-                write_byte(TYPE_LONG);
-                write_long(((PyLong) v).getValue());
-            } else if (v instanceof PyFloat) {
-                if (version == CURRENT_VERSION) {
-                    write_byte(TYPE_BINARY_FLOAT);
-                    write_binary_float((PyFloat) v);
-                } else {
-                    write_byte(TYPE_FLOAT);
-                    write_float((PyFloat) v);
-                }
-            } else if (v instanceof PyComplex) {
-                PyComplex x = (PyComplex) v;
-                if (version == CURRENT_VERSION) {
-                    write_byte(TYPE_BINARY_COMPLEX);
-                    write_binary_float(x.getReal());
-                    write_binary_float(x.getImag());
+            } else {
+                PyType vt = v.getType();
+                if (vt == PyInteger.TYPE) {
+                    write_byte(TYPE_INT);
+                    write_int(((PyInteger) v).asInt());
+                } else if (vt == PyLong.TYPE) {
+                    write_byte(TYPE_LONG);
+                    write_long(((PyLong) v).getValue());
+                } else if (vt == PyFloat.TYPE) {
+                    if (version == CURRENT_VERSION) {
+                        write_byte(TYPE_BINARY_FLOAT);
+                        write_binary_float((PyFloat) v);
+                    } else {
+                        write_byte(TYPE_FLOAT);
+                        write_float((PyFloat) v);
+                    }
+                } else if (vt == PyComplex.TYPE) {
+                    PyComplex x = (PyComplex) v;
+                    if (version == CURRENT_VERSION) {
+                        write_byte(TYPE_BINARY_COMPLEX);
+                        write_binary_float(x.getReal());
+                        write_binary_float(x.getImag());
+                    } else {
+                        write_byte(TYPE_COMPLEX);
+                        write_float(x.getReal());
+                        write_float(x.getImag());
+                    }
+                } else if (vt == PyUnicode.TYPE) {
+                    write_byte(TYPE_UNICODE);
+                    String buffer = ((PyUnicode) v).encode("utf-8").toString();
+                    write_int(buffer.length());
+                    write_string(buffer);
+                } else if (vt == PyString.TYPE) {
+                    // ignore interning
+                    write_byte(TYPE_STRING);
+                    write_int(v.__len__());
+                    write_string(v.toString());
+                } else if (vt == PyTuple.TYPE) {
+                    write_byte(TYPE_TUPLE);
+                    PyTuple t = (PyTuple) v;
+                    int n = t.__len__();
+                    write_int(n);
+                    for (int i = 0; i < n; i++) {
+                        write_object(t.__getitem__(i), depth + 1);
+                    }
+                } else if (vt == PyList.TYPE) {
+                    write_byte(TYPE_LIST);
+                    PyList list = (PyList) v;
+                    int n = list.__len__();
+                    write_int(n);
+                    for (int i = 0; i < n; i++) {
+                        write_object(list.__getitem__(i), depth + 1);
+                    }
+                } else if (vt == PyDictionary.TYPE) {
+                    write_byte(TYPE_DICT);
+                    PyDictionary dict = (PyDictionary) v;
+                    for (PyObject item : dict.iteritems().asIterable()) {
+                        PyTuple pair = (PyTuple) item;
+                        write_object(pair.__getitem__(0), depth + 1);
+                        write_object(pair.__getitem__(1), depth + 1);
+                    }
+                    write_object(null, depth + 1);
+                } else if (vt == PySet.TYPE || vt == PyFrozenSet.TYPE) {
+                    if (vt == PySet.TYPE) {
+                        write_byte(TYPE_SET);
+                    } else {
+                        write_byte(TYPE_FROZENSET);
+                    }
+                    int n = v.__len__();
+                    write_int(n);
+                    BaseSet set = (BaseSet) v;
+                    for (PyObject item : set.asIterable()) {
+                        write_object(item, depth + 1);
+                    }
+                } else if (vt == PyBytecode.TYPE) {
+                    PyBytecode code = (PyBytecode) v;
+                    write_byte(TYPE_CODE);
+                    write_int(code.co_argcount);
+                    write_int(code.co_nlocals);
+                    write_int(code.co_stacksize);
+                    write_int(code.co_flags.toBits());
+                    write_object(Py.newString(new String(code.co_code)), depth + 1);
+                    write_object(new PyTuple(code.co_consts), depth + 1);
+                    write_strings(code.co_names, depth + 1);
+                    write_strings(code.co_varnames, depth + 1);
+                    write_strings(code.co_freevars, depth + 1);
+                    write_strings(code.co_cellvars, depth + 1);
+                    write_object(Py.newString(code.co_name), depth + 1);
+                    write_int(code.co_firstlineno);
+                    write_object(Py.newString(new String(code.co_lnotab)), depth + 1);
                 } else {
-                    write_byte(TYPE_COMPLEX);
-                    write_float(x.getReal());
-                    write_float(x.getImag());
-                }
-            } else if (v instanceof PyUnicode) {
-                write_byte(TYPE_UNICODE);
-                String buffer = ((PyUnicode) v).encode("utf-8").toString();
-                write_int(buffer.length());
-                write_string(buffer);
-            } else if (v instanceof PyString) {
-                // ignore interning
-                write_byte(TYPE_STRING);
-                write_int(v.__len__());
-                write_string(v.toString());
-            } else if (v instanceof PyTuple) {
-                write_byte(TYPE_TUPLE);
-                PyTuple t = (PyTuple) v;
-                int n = t.__len__();
-                write_int(n);
-                for (int i = 0; i < n; i++) {
-                    write_object(t.__getitem__(i), depth + 1);
+                    // Try to get a simple byte-oriented buffer
+                    try (PyBuffer buf = ((BufferProtocol) v).getBuffer(PyBUF.SIMPLE)) {
+                        // ... and treat those bytes as a String
+                        write_byte(TYPE_STRING);
+                        write_int(v.__len__());
+                        write_string(buf.toString());
+                    } catch (ClassCastException | PyException e) {
+                        // Does not implement BufferProtocol (in simple byte form).
+                        throw Py.ValueError("unmarshallable object");
+                    }
                 }
-            } else if (v instanceof PyList) {
-                write_byte(TYPE_LIST);
-                PyList list = (PyList) v;
-                int n = list.__len__();
-                write_int(n);
-                for (int i = 0; i < n; i++) {
-                    write_object(list.__getitem__(i), depth + 1);
-                }
-            } else if (v instanceof PyDictionary) {
-                write_byte(TYPE_DICT);
-                PyDictionary dict = (PyDictionary) v;
-                for (PyObject item : dict.iteritems().asIterable()) {
-                    PyTuple pair = (PyTuple) item;
-                    write_object(pair.__getitem__(0), depth + 1);
-                    write_object(pair.__getitem__(1), depth + 1);
-                }
-                write_object(null, depth + 1);
-            } else if (v instanceof BaseSet) {
-                if (v instanceof PySet) {
-                    write_byte(TYPE_SET);
-                } else {
-                    write_byte(TYPE_FROZENSET);
-                }
-                int n = v.__len__();
-                write_int(n);
-                BaseSet set = (BaseSet) v;
-                for (PyObject item : set.asIterable()) {
-                    write_object(item, depth + 1);
-                }
-            } else if (v instanceof PyBytecode) {
-                PyBytecode code = (PyBytecode) v;
-                write_byte(TYPE_CODE);
-                write_int(code.co_argcount);
-                write_int(code.co_nlocals);
-                write_int(code.co_stacksize);
-                write_int(code.co_flags.toBits());
-                write_object(Py.newString(new String(code.co_code)), depth + 1);
-                write_object(new PyTuple(code.co_consts), depth + 1);
-                write_strings(code.co_names, depth + 1);
-                write_strings(code.co_varnames, depth + 1);
-                write_strings(code.co_freevars, depth + 1);
-                write_strings(code.co_cellvars, depth + 1);
-                write_object(Py.newString(code.co_name), depth + 1);
-                write_int(code.co_firstlineno);
-                write_object(Py.newString(new String(code.co_lnotab)), depth + 1);
-            } else {
-                throw Py.ValueError("unmarshallable object");
             }
             depth--;
         }
@@ -327,21 +344,25 @@
         }
 
         private BigInteger read_long() {
-            int size = read_int();
-            int sign = 1;
-            if (size < 0) {
-                sign = -1;
+            BigInteger result = BigInteger.ZERO;
+            boolean negative = false;
+            int digit = 0, size = read_int();
+            if (size == 0) {
+                return result;
+            } else if (size < 0) {
+                negative = true;
                 size = -size;
             }
-            BigInteger result = BigInteger.ZERO;
             for (int i = 0; i < size; i++) {
-                String digits = String.valueOf(read_short());
-                result = result.or(new BigInteger(digits).shiftLeft(i * 15));
+                if ((digit = read_short()) < 0) {
+                    throw badMarshalData("digit out of range in long");
+                }
+                result = result.or(BigInteger.valueOf(digit).shiftLeft(i * 15));
             }
-            if (sign < 0) {
-                result = result.negate();
+            if (digit == 0) {
+                throw badMarshalData("unnormalized long data");
             }
-            return result;
+            return negative ? result.negate() : result;
         }
 
         private double read_float() {
@@ -356,7 +377,7 @@
         private PyObject read_object_notnull(int depth) {
             PyObject v = read_object(depth);
             if (v == null) {
-                throw Py.ValueError("bad marshal data");
+                throw badMarshalData(null);
             }
             return v;
         }
@@ -451,7 +472,7 @@
                 case TYPE_TUPLE: {
                     int n = read_int();
                     if (n < 0) {
-                        throw Py.ValueError("bad marshal data");
+                        throw badMarshalData(null);
                     }
                     PyObject items[] = new PyObject[n];
                     for (int i = 0; i < n; i++) {
@@ -463,7 +484,7 @@
                 case TYPE_LIST: {
                     int n = read_int();
                     if (n < 0) {
-                        throw Py.ValueError("bad marshal data");
+                        throw badMarshalData(null);
                     }
                     PyObject items[] = new PyObject[n];
                     for (int i = 0; i < n; i++) {
@@ -528,10 +549,18 @@
                 }
 
                 default:
-                    throw Py.ValueError("bad marshal data");
+                    throw badMarshalData("unknown type code");
             }
         }
 
+        /** Helper returning "bad marshal data" or "bad marshal data (<reason>)". */
+        private static PyException badMarshalData(String reason) {
+            StringBuilder msg = (new StringBuilder(60)).append("bad marshal data");
+            if (reason != null) {
+                msg.append(" (").append(reason).append(')');
+            }
+            return Py.ValueError(msg.toString());
+        }
 
         /* Traverseproc implementation */
         @Override

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Sat Mar 23 04:50:15 2019
From: jython-checkins at python.org (jeff.allen)
Date: Sat, 23 Mar 2019 08:50:15 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Update_test=5Fmarshal_from?=
 =?utf-8?q?_lib-python_and_add_some_skips=2E?=
Message-ID: <20190323085015.1.17523570691A20C8@mg.python.org>

https://hg.python.org/jython/rev/dc308813c14f
changeset:   8227:dc308813c14f
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Fri Mar 22 07:41:06 2019 +0000
summary:
  Update test_marshal from lib-python and add some skips.

This is mainly so we have a test of the fix for bjo #2077. Also corrects
spelling of Andrew Preview.

files:
  Lib/test/test_marshal.py |  120 ++++++++++++++++++++++----
  1 files changed, 98 insertions(+), 22 deletions(-)


diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py
--- a/Lib/test/test_marshal.py
+++ b/Lib/test/test_marshal.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python
-# -*- coding: iso-8859-1 -*-
+# -*- coding: utf-8 -*-
 
-from __future__ import with_statement
 from test import test_support
 import marshal
 import sys
@@ -118,7 +117,7 @@
 
 class StringTestCase(unittest.TestCase):
     def test_unicode(self):
-        for s in [u"", u"Andr? Previn", u"abc", u" "*10000]:
+        for s in [u"", u"Andr?? Previn", u"abc", u" "*10000]:
             new = marshal.loads(marshal.dumps(s))
             self.assertEqual(s, new)
             self.assertEqual(type(s), type(new))
@@ -128,7 +127,7 @@
         os.unlink(test_support.TESTFN)
 
     def test_string(self):
-        for s in ["", "Andr? Previn", "abc", " "*10000]:
+        for s in ["", "Andr?? Previn", "abc", " "*10000]:
             new = marshal.loads(marshal.dumps(s))
             self.assertEqual(s, new)
             self.assertEqual(type(s), type(new))
@@ -137,27 +136,29 @@
             self.assertEqual(type(s), type(new))
         os.unlink(test_support.TESTFN)
 
-    if not test_support.is_jython:
-        def test_buffer(self):
-            for s in ["", "Andr? Previn", "abc", " "*10000]:
+    @unittest.skipIf(test_support.is_jython, "FIXME: bjo #2744 buffer not supported")
+    def test_buffer(self):
+        for s in ["", "Andr?? Previn", "abc", " "*10000]:
+            with test_support.check_py3k_warnings(("buffer.. not supported",
+                                                     DeprecationWarning)):
                 b = buffer(s)
-                new = marshal.loads(marshal.dumps(b))
-                self.assertEqual(s, new)
-                new = roundtrip(b)
-                self.assertEqual(s, new)
-            os.unlink(test_support.TESTFN)
+            new = marshal.loads(marshal.dumps(b))
+            self.assertEqual(s, new)
+            new = roundtrip(b)
+            self.assertEqual(s, new)
+        os.unlink(test_support.TESTFN)
 
 class ExceptionTestCase(unittest.TestCase):
     def test_exceptions(self):
         new = marshal.loads(marshal.dumps(StopIteration))
         self.assertEqual(StopIteration, new)
 
+ at unittest.skipIf(test_support.is_jython, "requires PBC compilation back-end")
 class CodeTestCase(unittest.TestCase):
-    if not test_support.is_jython: # XXX - need to use the PBC compilation backend, which doesn't exist yet
-        def test_code(self):
-            co = ExceptionTestCase.test_exceptions.func_code
-            new = marshal.loads(marshal.dumps(co))
-            self.assertEqual(co, new)
+    def test_code(self):
+        co = ExceptionTestCase.test_exceptions.func_code
+        new = marshal.loads(marshal.dumps(co))
+        self.assertEqual(co, new)
 
 class ContainerTestCase(unittest.TestCase):
     d = {'astring': 'foo at bar.baz.spam',
@@ -167,7 +168,7 @@
          'alist': ['.zyx.41'],
          'atuple': ('.zyx.41',)*10,
          'aboolean': False,
-         'aunicode': u"Andr? Previn"
+         'aunicode': u"Andr?? Previn"
          }
     def test_dict(self):
         new = marshal.loads(marshal.dumps(self.d))
@@ -197,7 +198,7 @@
             t = constructor(self.d.keys())
             new = marshal.loads(marshal.dumps(t))
             self.assertEqual(t, new)
-            self.assert_(isinstance(new, constructor))
+            self.assertTrue(isinstance(new, constructor))
             self.assertNotEqual(id(t), id(new))
             new = roundtrip(t)
             self.assertEqual(t, new)
@@ -215,8 +216,8 @@
 
     def test_version_argument(self):
         # Python 2.4.0 crashes for any call to marshal.dumps(x, y)
-        self.assertEquals(marshal.loads(marshal.dumps(5, 0)), 5)
-        self.assertEquals(marshal.loads(marshal.dumps(5, 1)), 5)
+        self.assertEqual(marshal.loads(marshal.dumps(5, 0)), 5)
+        self.assertEqual(marshal.loads(marshal.dumps(5, 1)), 5)
 
     def test_fuzz(self):
         # simple test that it's at least not *totally* trivial to
@@ -251,6 +252,79 @@
         last.append([0])
         self.assertRaises(ValueError, marshal.dumps, head)
 
+    @unittest.skipIf(test_support.is_jython, "FIXME: bjo #2077 ValueError not raised")
+    def test_exact_type_match(self):
+        # Former bug:
+        #   >>> class Int(int): pass
+        #   >>> type(loads(dumps(Int())))
+        #   <type 'int'>
+        for typ in (int, long, float, complex, tuple, list, dict, set, frozenset):
+            # Note: str and unicode subclasses are not tested because they get handled
+            # by marshal's routines for objects supporting the buffer API.
+            subtyp = type('subtyp', (typ,), {})
+            self.assertRaises(ValueError, marshal.dumps, subtyp())
+
+    # Issue #1792 introduced a change in how marshal increases the size of its
+    # internal buffer; this test ensures that the new code is exercised.
+    def test_large_marshal(self):
+        size = int(1e6)
+        testString = 'abc' * size
+        marshal.dumps(testString)
+
+    @unittest.skipIf(test_support.is_jython, "FIXME: bjo #2077 ValueError not raised")
+    def test_invalid_longs(self):
+        # Issue #7019: marshal.loads shouldn't produce unnormalized PyLongs
+        invalid_string = 'l\x02\x00\x00\x00\x00\x00\x00\x00'
+        self.assertRaises(ValueError, marshal.loads, invalid_string)
+
+LARGE_SIZE = 2**31
+character_size = 4 if sys.maxunicode > 0xFFFF else 2
+pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4
+
+ at unittest.skipIf(LARGE_SIZE > sys.maxsize, "requires larger sys.maxsize")
+class LargeValuesTestCase(unittest.TestCase):
+    def check_unmarshallable(self, data):
+        f = open(test_support.TESTFN, 'wb')
+        self.addCleanup(test_support.unlink, test_support.TESTFN)
+        with f:
+            self.assertRaises(ValueError, marshal.dump, data, f)
+
+    @test_support.precisionbigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False)
+    def test_string(self, size):
+        self.check_unmarshallable('x' * size)
+
+    @test_support.precisionbigmemtest(size=LARGE_SIZE,
+            memuse=character_size, dry_run=False)
+    def test_unicode(self, size):
+        self.check_unmarshallable(u'x' * size)
+
+    @test_support.precisionbigmemtest(size=LARGE_SIZE,
+            memuse=pointer_size, dry_run=False)
+    def test_tuple(self, size):
+        self.check_unmarshallable((None,) * size)
+
+    @test_support.precisionbigmemtest(size=LARGE_SIZE,
+            memuse=pointer_size, dry_run=False)
+    def test_list(self, size):
+        self.check_unmarshallable([None] * size)
+
+    @test_support.precisionbigmemtest(size=LARGE_SIZE,
+            memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
+            dry_run=False)
+    def test_set(self, size):
+        self.check_unmarshallable(set(range(size)))
+
+    @test_support.precisionbigmemtest(size=LARGE_SIZE,
+            memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
+            dry_run=False)
+    def test_frozenset(self, size):
+        self.check_unmarshallable(frozenset(range(size)))
+
+    @test_support.precisionbigmemtest(size=LARGE_SIZE, memuse=1, dry_run=False)
+    def test_bytearray(self, size):
+        self.check_unmarshallable(bytearray(size))
+
+
 def test_main():
     test_support.run_unittest(IntTestCase,
                               FloatTestCase,
@@ -258,7 +332,9 @@
                               CodeTestCase,
                               ContainerTestCase,
                               ExceptionTestCase,
-                              BugsTestCase)
+                              BugsTestCase,
+                              LargeValuesTestCase,
+                             )
 
 if __name__ == "__main__":
     test_main()

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Thu Mar 28 04:57:03 2019
From: jython-checkins at python.org (jeff.allen)
Date: Thu, 28 Mar 2019 08:57:03 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Add_skip_for_test=5Fsetdef?=
 =?utf-8?q?ault=5Fatomic_in_tests_of_ConcurrentHashMap=2E?=
Message-ID: <20190328085703.1.1C15892B7FDCF6F2@mg.python.org>

https://hg.python.org/jython/rev/10d69614720a
changeset:   8232:10d69614720a
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Thu Mar 28 08:48:25 2019 +0000
summary:
  Add skip for test_setdefault_atomic in tests of ConcurrentHashMap.

This test, previously skipped by accident, fails on Java 7. Issue #2711 refers.

files:
  Lib/test/test_dict_jy.py |  4 +++-
  1 files changed, 3 insertions(+), 1 deletions(-)


diff --git a/Lib/test/test_dict_jy.py b/Lib/test/test_dict_jy.py
--- a/Lib/test/test_dict_jy.py
+++ b/Lib/test/test_dict_jy.py
@@ -381,10 +381,12 @@
     @unittest.skip("FIXME: see bjo #2746")
     def test_repr_value_None(self): pass # defining here only so we can skip it
 
-
 class JavaConcurrentHashMapDictTest(JavaDictTest):
     _class = ConcurrentHashMap
 
+    @unittest.skip("FIXME: bjo #2711 affects ConcurrentHashMap in all Java versions.")
+    def test_setdefault_atomic(self): pass # defining here only so we can skip it
+
     @unittest.skip("FIXME: see bjo #2746")
     def test_has_key(self): pass # defining here only so we can skip it
 

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Thu Mar 28 04:57:03 2019
From: jython-checkins at python.org (jeff.allen)
Date: Thu, 28 Mar 2019 08:57:03 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Avoid_generating_Unicode_c?=
 =?utf-8?q?haracters_in_repr_of_JavaProxyMap_=28fixes_bjo_=232649=29=2E?=
Message-ID: <20190328085703.1.D3A856214E7671AD@mg.python.org>

https://hg.python.org/jython/rev/d24e20f4fc1e
changeset:   8231:d24e20f4fc1e
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Wed Mar 27 22:25:26 2019 +0000
summary:
  Avoid generating Unicode characters in repr of JavaProxyMap (fixes bjo #2649).

In order to pass the infinite recursion test, it is also necessary to have
enterRepr cope with proxies.

files:
  Lib/test/test_dict_jy.py              |   7 +-
  src/org/python/core/JavaProxyMap.java |  36 +++++++++-----
  src/org/python/core/ThreadState.java  |  10 ++--
  3 files changed, 30 insertions(+), 23 deletions(-)


diff --git a/Lib/test/test_dict_jy.py b/Lib/test/test_dict_jy.py
--- a/Lib/test/test_dict_jy.py
+++ b/Lib/test/test_dict_jy.py
@@ -290,19 +290,18 @@
 
     # Some variants with unicode keys
 
-    @unittest.skip("FIXME: failing test for bjo #2649")
     def test_repr_unicode(self):
         d = self._make_dict({})
         d[u'3\uc6d4'] = 2
-        self.assertEqual(repr(d), "{u'3\uc6d4': 2}")
+        self.assertEqual(repr(d), "{u'3\\uc6d4': 2}")
 
         d = self._make_dict({})
         d[2] = u'\u039c\u03ac\u03c1\u03c4\u03b9\u03bf\u03c2'
-        self.assertEqual(repr(d), "{2: u'\u039c\u03ac\u03c1\u03c4\u03b9\u03bf\u03c2'}")
+        self.assertEqual(repr(d), "{2: u'\\u039c\\u03ac\\u03c1\\u03c4\\u03b9\\u03bf\\u03c2'}")
 
         d = self._make_dict({})
         d[u'\uc6d4'] = d
-        self.assertEqual(repr(d), "{u'\uc6d4': {...}}")
+        self.assertEqual(repr(d), "{u'\\uc6d4': {...}}")
 
     def test_fromkeys_unicode(self):
         super(JavaDictTest, self).test_fromkeys()
diff --git a/src/org/python/core/JavaProxyMap.java b/src/org/python/core/JavaProxyMap.java
--- a/src/org/python/core/JavaProxyMap.java
+++ b/src/org/python/core/JavaProxyMap.java
@@ -129,23 +129,31 @@
         }
     };
     private static final PyBuiltinMethodNarrow mapReprProxy = new MapMethod("__repr__", 0) {
+
         @Override
         public PyObject __call__() {
-            StringBuilder repr = new StringBuilder("{");
-            for (Map.Entry<Object, Object> entry : asMap().entrySet()) {
-                Object jkey = entry.getKey();
-                Object jval = entry.getValue();
-                repr.append(jkey.toString());
-                repr.append(": ");
-                repr.append(jval == asMap() ? "{...}" : (jval == null ? "None" : jval.toString()));
-                repr.append(", ");
+            ThreadState ts = Py.getThreadState();
+            if (!ts.enterRepr(self)) {
+                return Py.newString("{...}");
+            } else {
+                StringBuilder repr = new StringBuilder("{");
+                boolean first = true;
+                for (Map.Entry<Object, Object> entry : asMap().entrySet()) {
+                    if (first) {
+                        first=false;
+                    } else {
+                        repr.append(", ");
+                    }
+                    PyObject key = Py.java2py(entry.getKey());
+                    repr.append(key.__repr__().toString());
+                    repr.append(": ");
+                    PyObject value = Py.java2py(entry.getValue());
+                    repr.append(value.__repr__().toString());
+                }
+                repr.append("}");
+                ts.exitRepr(self);
+                return Py.newString(repr.toString());
             }
-            int lastindex = repr.lastIndexOf(", ");
-            if (lastindex > -1) {
-                repr.delete(lastindex, lastindex + 2);
-            }
-            repr.append("}");
-            return new PyString(repr.toString());
         }
     };
     private static final PyBuiltinMethodNarrow mapEqProxy = new MapMethod("__eq__", 1) {
diff --git a/src/org/python/core/ThreadState.java b/src/org/python/core/ThreadState.java
--- a/src/org/python/core/ThreadState.java
+++ b/src/org/python/core/ThreadState.java
@@ -38,19 +38,19 @@
             systemStateRef = new PySystemStateRef(systemState, this);
         }
     }
-    
+
     public PySystemState getSystemState() {
         PySystemState systemState = systemStateRef == null ? null : systemStateRef.get();
-        return systemState == null ? Py.defaultSystemState : systemState; 
+        return systemState == null ? Py.defaultSystemState : systemState;
     }
-    
+
     public boolean enterRepr(PyObject obj) {
         if (reprStack == null) {
             reprStack = new PyList(new PyObject[] {obj});
             return true;
         }
         for (int i = reprStack.size() - 1; i >= 0; i--) {
-            if (obj == reprStack.pyget(i)) {
+            if (obj._is(reprStack.pyget(i)).__nonzero__()) {
                 return false;
             }
         }
@@ -63,7 +63,7 @@
             return;
         }
         for (int i = reprStack.size() - 1; i >= 0; i--) {
-            if (reprStack.pyget(i) == obj) {
+            if (obj._is(reprStack.pyget(i)).__nonzero__()) {
                 reprStack.delRange(i, reprStack.size());
             }
         }

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Thu Mar 28 04:57:03 2019
From: jython-checkins at python.org (jeff.allen)
Date: Thu, 28 Mar 2019 08:57:03 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Improve_coverage_of_map_ty?=
 =?utf-8?q?pes_available_from_Java_in_test=5Fdict=5Fjy=2E?=
Message-ID: <20190328085703.1.28D451FF3B88F19B@mg.python.org>

https://hg.python.org/jython/rev/cb8787723e59
changeset:   8230:cb8787723e59
user:        Jeff Allen <ja.py at farowl.co.uk>
date:        Mon Mar 25 06:06:52 2019 +0000
summary:
  Improve coverage of map types available from Java in test_dict_jy.

Adds test of repr when dictionaries contain unicode keys or values (bjo #2649).
Extends all dictionary tests to 4 major Java containers, requiring some skips
for failing tests now lodged as bjo #2746.

files:
  Lib/test/test_dict_jy.py |  67 ++++++++++++++++++++++++++-
  1 files changed, 63 insertions(+), 4 deletions(-)


diff --git a/Lib/test/test_dict_jy.py b/Lib/test/test_dict_jy.py
--- a/Lib/test/test_dict_jy.py
+++ b/Lib/test/test_dict_jy.py
@@ -245,12 +245,12 @@
     type2test = Hashtable
 
 class JavaConcurrentHashMapTest(JavaIntegrationTest):
-    type2test = HashMap
+    type2test = ConcurrentHashMap
 
 
 class JavaDictTest(test_dict.DictTest):
 
-    _class = HashMap
+    _class = dict
 
     def test_copy_java_hashtable(self):
         x = Hashtable()
@@ -288,6 +288,27 @@
         with self.assertRaises(AssertionError):
             prop(self._make_dict(a), b)
 
+    # Some variants with unicode keys
+
+    @unittest.skip("FIXME: failing test for bjo #2649")
+    def test_repr_unicode(self):
+        d = self._make_dict({})
+        d[u'3\uc6d4'] = 2
+        self.assertEqual(repr(d), "{u'3\uc6d4': 2}")
+
+        d = self._make_dict({})
+        d[2] = u'\u039c\u03ac\u03c1\u03c4\u03b9\u03bf\u03c2'
+        self.assertEqual(repr(d), "{2: u'\u039c\u03ac\u03c1\u03c4\u03b9\u03bf\u03c2'}")
+
+        d = self._make_dict({})
+        d[u'\uc6d4'] = d
+        self.assertEqual(repr(d), "{u'\uc6d4': {...}}")
+
+    def test_fromkeys_unicode(self):
+        super(JavaDictTest, self).test_fromkeys()
+        self.assertEqual(self._class.fromkeys(u'\U00010840\U00010841\U00010842'),
+                         {u'\U00010840':None, u'\U00010841':None, u'\U00010842':None})
+
     # NOTE: when comparing dictionaries below exclusively in Java
     # space, keys like 1 and 1L are different objects. Only when they
     # are brought into Python space by Py.java2py, as is needed when
@@ -327,7 +348,7 @@
         self.assertGreater({1L: 2L, 3L: 4L}, self._make_dict({1: 2}))
 
 
-class PyStringMapTest(test_dict.DictTest):
+class PyStringMapDictTest(test_dict.DictTest):
     # __dict__ for objects uses PyStringMap for historical reasons, so
     # we have to test separately
 
@@ -343,6 +364,38 @@
         return newdict
 
 
+class JavaHashMapDictTest(JavaDictTest):
+    _class = HashMap
+
+class JavaLinkedHashMapDictTest(JavaDictTest):
+    _class = LinkedHashMap
+
+class JavaHashtableDictTest(JavaDictTest):
+    _class = Hashtable
+
+    @unittest.skip("FIXME: see bjo #2746")
+    def test_has_key(self): pass # defining here only so we can skip it
+
+    @unittest.skip("FIXME: see bjo #2746")
+    def test_keys(self): pass # defining here only so we can skip it
+
+    @unittest.skip("FIXME: see bjo #2746")
+    def test_repr_value_None(self): pass # defining here only so we can skip it
+
+
+class JavaConcurrentHashMapDictTest(JavaDictTest):
+    _class = ConcurrentHashMap
+
+    @unittest.skip("FIXME: see bjo #2746")
+    def test_has_key(self): pass # defining here only so we can skip it
+
+    @unittest.skip("FIXME: see bjo #2746")
+    def test_keys(self): pass # defining here only so we can skip it
+
+    @unittest.skip("FIXME: see bjo #2746")
+    def test_repr_value_None(self): pass # defining here only so we can skip it
+
+
 def test_main():
     test_support.run_unittest(
         DictInitTest,
@@ -354,7 +407,13 @@
         JavaConcurrentHashMapTest,
         JavaHashtableTest,
         JavaDictTest,
-        PyStringMapTest)
+        PyStringMapDictTest,
+        JavaHashMapDictTest,
+        JavaLinkedHashMapDictTest,
+        JavaHashtableDictTest,
+        JavaConcurrentHashMapDictTest,
+    )
+
 
 if __name__ == '__main__':
     test_main()

-- 
Repository URL: https://hg.python.org/jython