[Python-checkins] distutils2: merge the installer
tarek.ziade
python-checkins at python.org
Sun Jan 30 10:43:57 CET 2011
tarek.ziade pushed 931794fdbd09 to distutils2:
http://hg.python.org/distutils2/rev/931794fdbd09
changeset: 925:931794fdbd09
parent: 898:06870bf6581a
parent: 924:4a41a15d1f9f
user: Alexis Metaireau <alexis at notmyidea.org>
date: Sat Jan 29 10:34:44 2011 +0100
summary:
merge the installer
files:
distutils2/_backport/shutil.py
distutils2/index/dist.py
distutils2/index/simple.py
distutils2/install.py
distutils2/run.py
distutils2/tests/pypi_server.py
diff --git a/distutils2/_backport/shutil.py b/distutils2/_backport/shutil.py
--- a/distutils2/_backport/shutil.py
+++ b/distutils2/_backport/shutil.py
@@ -568,3 +568,72 @@
os.chdir(save_cwd)
return filename
+
+
+def _ensure_directory(path):
+ """Ensure that the parent directory of `path` exists"""
+ dirname = os.path.dirname(path)
+ if not os.path.isdir(dirname):
+ os.makedirs(dirname)
+
+def _unpack_zipfile(filename, extract_dir):
+ try:
+ import zipfile
+ except ImportError:
+ raise ReadError('zlib not supported, cannot unpack this archive.')
+
+ if not zipfile.is_zipfile(filename):
+ raise ReadError("%s is not a zip file" % filename)
+
+ zip = zipfile.ZipFile(filename)
+ try:
+ for info in zip.infolist():
+ name = info.filename
+ if name.startswith('/') or '..' in name:
+ continue
+
+ target = os.path.join(extract_dir, *name.split('/'))
+ if not target:
+ continue
+
+ _ensure_directory(target)
+ if not name.endswith('/'):
+ # file
+ data = zip.read(info.filename)
+ f = open(target,'wb')
+ try:
+ f.write(data)
+ finally:
+ f.close()
+ del data
+ finally:
+ zip.close()
+
+def _unpack_tarfile(filename, extract_dir):
+ try:
+ tarobj = tarfile.open(filename)
+ except tarfile.TarError:
+ raise ReadError(
+ "%s is not a compressed or uncompressed tar file" % filename)
+ try:
+ tarobj.extractall(extract_dir)
+ finally:
+ tarobj.close()
+
+
+_UNPACKERS = (
+ (['.tar.gz', '.tgz', '.tar'], _unpack_tarfile),
+ (['.zip', '.egg'], _unpack_zipfile))
+
+
+def unpack_archive(filename, extract_dir=None):
+ if extract_dir is None:
+ extract_dir = os.path.dirname(filename)
+
+ for formats, func in _UNPACKERS:
+ for format in formats:
+ if filename.endswith(format):
+ func(filename, extract_dir)
+ return extract_dir
+
+ raise ValueError('Unknown archive format: %s' % filename)
diff --git a/distutils2/index/dist.py b/distutils2/index/dist.py
--- a/distutils2/index/dist.py
+++ b/distutils2/index/dist.py
@@ -17,19 +17,19 @@
import urllib
import urlparse
import zipfile
-
try:
import hashlib
except ImportError:
from distutils2._backport import hashlib
+from distutils2._backport.shutil import unpack_archive
from distutils2.errors import IrrationalVersionError
from distutils2.index.errors import (HashDoesNotMatch, UnsupportedHashName,
CantParseArchiveName)
from distutils2.version import (suggest_normalized_version, NormalizedVersion,
get_version_predicate)
from distutils2.metadata import DistributionMetadata
-from distutils2.util import untar_file, unzip_file, splitext
+from distutils2.util import splitext
__all__ = ['ReleaseInfo', 'DistInfo', 'ReleasesList', 'get_infos_from_url']
@@ -206,6 +206,7 @@
__hash__ = object.__hash__
+
class DistInfo(IndexReference):
"""Represents a distribution retrieved from an index (sdist, bdist, ...)
"""
@@ -313,17 +314,8 @@
filename = self.download()
content_type = mimetypes.guess_type(filename)[0]
+ self._unpacked_dir = unpack_archive(filename)
- if (content_type == 'application/zip'
- or filename.endswith('.zip')
- or filename.endswith('.pybundle')
- or zipfile.is_zipfile(filename)):
- unzip_file(filename, path, flatten=not filename.endswith('.pybundle'))
- elif (content_type == 'application/x-gzip'
- or tarfile.is_tarfile(filename)
- or splitext(filename)[1].lower() in ('.tar', '.tar.gz', '.tar.bz2', '.tgz', '.tbz')):
- untar_file(filename, path)
- self._unpacked_dir = path
return self._unpacked_dir
def _check_md5(self, filename):
diff --git a/distutils2/index/xmlrpc.py b/distutils2/index/xmlrpc.py
--- a/distutils2/index/xmlrpc.py
+++ b/distutils2/index/xmlrpc.py
@@ -127,10 +127,17 @@
return release
def get_metadata(self, project_name, version):
- """Retreive project metadatas.
+ """Retrieve project metadata.
Return a ReleaseInfo object, with metadata informations filled in.
"""
+ # to be case-insensitive
+ projects = [d['name'] for d in
+ self.proxy.search({'name': project_name})
+ if d['name'].lower() == project_name]
+ if len(projects) > 0:
+ project_name = projects[0]
+
metadata = self.proxy.release_data(project_name, version)
project = self._get_project(project_name)
if version not in project.get_versions():
diff --git a/distutils2/install.py b/distutils2/install.py
--- a/distutils2/install.py
+++ b/distutils2/install.py
@@ -1,14 +1,16 @@
from tempfile import mkdtemp
-import logging
import shutil
import os
import errno
import itertools
+from distutils2 import logger
from distutils2._backport.pkgutil import get_distributions
+from distutils2._backport.sysconfig import get_config_var
from distutils2.depgraph import generate_graph
from distutils2.index import wrapper
from distutils2.index.errors import ProjectNotFound, ReleaseNotFound
+from distutils2.version import get_version_predicate
"""Provides installations scripts.
@@ -53,7 +55,63 @@
else:
raise e
os.rename(old, new)
- yield(old, new)
+ yield (old, new)
+
+
+def _run_d1_install(archive_dir, path):
+ # backward compat: using setuptools or plain-distutils
+ cmd = '%s setup.py install --root=%s --record=%s'
+ setup_py = os.path.join(archive_dir, 'setup.py')
+ if 'setuptools' in open(setup_py).read():
+ cmd += ' --single-version-externally-managed'
+
+ # how to place this file in the egg-info dir
+ # for non-distutils2 projects ?
+ record_file = os.path.join(archive_dir, 'RECORD')
+ os.system(cmd % (sys.executable, path, record_file))
+ if not os.path.exists(record_file):
+ raise ValueError('Failed to install.')
+ return open(record_file).read().split('\n')
+
+
+def _run_d2_install(archive_dir, path):
+ # using our own install command
+ raise NotImplementedError()
+
+
+def _install_dist(dist, path):
+ """Install a distribution into a path.
+
+ This:
+
+ * unpack the distribution
+ * copy the files in "path"
+ * determine if the distribution is distutils2 or distutils1.
+ """
+ where = dist.unpack(archive)
+
+ # get into the dir
+ archive_dir = None
+ for item in os.listdir(where):
+ fullpath = os.path.join(where, item)
+ if os.path.isdir(fullpath):
+ archive_dir = fullpath
+ break
+
+ if archive_dir is None:
+ raise ValueError('Cannot locate the unpacked archive')
+
+ # install
+ old_dir = os.getcwd()
+ os.chdir(archive_dir)
+ try:
+ # distutils2 or distutils1 ?
+ if 'setup.py' in os.listdir(archive_dir):
+ return _run_d1_install(archive_dir, path)
+ else:
+ return _run_d2_install(archive_dir, path)
+ finally:
+ os.chdir(old_dir)
def install_dists(dists, path=None):
@@ -65,19 +123,23 @@
Return a list of installed files.
:param dists: distributions to install
- :param path: base path to install distribution on
+ :param path: base path to install distribution in
"""
if not path:
path = mkdtemp()
installed_dists, installed_files = [], []
for d in dists:
+ logger.info('Installing %s %s' % (d.name, d.version))
try:
- installed_files.extend(d.install(path))
+ installed_files.extend(_install_dist(d, path))
installed_dists.append(d)
except Exception, e :
+ logger.info('Failed. %s' % str(e))
+
+ # reverting
for d in installed_dists:
- d.uninstall()
+ _uninstall(d)
raise e
return installed_files
@@ -123,16 +185,26 @@
try:
if install:
installed_files = install_dists(install, install_path) # install to tmp first
- for files in temp_files.itervalues():
- for old, new in files:
- os.remove(new)
- except Exception,e:
+ except:
# if an error occurs, put back the files in the good place.
- for files in temp_files.itervalues():
+ for files in temp_files.values():
for old, new in files:
shutil.move(new, old)
+ # now re-raising
+ raise
+
+ # we can remove them for good
+ for files in temp_files.values():
+ for old, new in files:
+ os.remove(new)
+
+
+def _get_setuptools_deps(release):
+ # NotImplementedError
+ pass
+
def get_infos(requirements, index=None, installed=None, prefer_final=True):
"""Return the informations on what's going to be installed and upgraded.
@@ -154,13 +226,35 @@
Conflict contains all the conflicting distributions, if there is a
conflict.
"""
+ from ipdb import set_trace
+ set_trace()
+ if not installed:
+ logger.info('Reading installed distributions')
+ installed = get_distributions(use_egg_info=True)
+
+ infos = {'install': [], 'remove': [], 'conflict': []}
+ # Is a compatible version of the project is already installed ?
+ predicate = get_version_predicate(requirements)
+ found = False
+ installed = list(installed)
+ for installed_project in installed:
+ # is it a compatible project ?
+ if predicate.name.lower() != installed_project.name.lower():
+ continue
+ found = True
+ logger.info('Found %s %s' % (installed_project.name,
+ installed_project.metadata.version))
+ if predicate.match(installed_project.metadata.version):
+ return infos
+
+ break
+
+ if not found:
+ logger.info('Project not installed.')
if not index:
index = wrapper.ClientWrapper()
- if not installed:
- installed = get_distributions(use_egg_info=True)
-
# Get all the releases that match the requirements
try:
releases = index.get_releases(requirements)
@@ -170,28 +264,34 @@
# Pick up a release, and try to get the dependency tree
release = releases.get_last(requirements, prefer_final=prefer_final)
- # Iter since we found something without conflicts
+ if release is None:
+ logger.info('Could not find a matching project')
+ return infos
+
+ # this works for Metadata 1.2
metadata = release.fetch_metadata()
- # Get the distributions already_installed on the system
- # and add the one we want to install
+ # for earlier, we need to build setuptools deps if any
+ if 'requires_dist' not in metadata:
+ deps = _get_setuptools_deps(release)
+ else:
+ deps = metadata['requires_dist']
distributions = itertools.chain(installed, [release])
depgraph = generate_graph(distributions)
# Store all the already_installed packages in a list, in case of rollback.
- infos = {'install': [], 'remove': [], 'conflict': []}
-
# Get what the missing deps are
- for dists in depgraph.missing.itervalues():
- if dists:
- logging.info("missing dependencies found, installing them")
- # we have missing deps
- for dist in dists:
- _update_infos(infos, get_infos(dist, index, installed))
+ dists = depgraph.missing[release]
+ if dists:
+ logger.info("missing dependencies found, installing them")
+ # we have missing deps
+ for dist in dists:
+ _update_infos(infos, get_infos(dist, index, installed))
# Fill in the infos
existing = [d for d in installed if d.name == release.name]
+
if existing:
infos['remove'].append(existing[0])
infos['conflict'].extend(depgraph.reverse_list[existing[0]])
@@ -203,7 +303,7 @@
"""extends the lists contained in the `info` dict with those contained
in the `new_info` one
"""
- for key, value in infos.iteritems():
+ for key, value in infos.items():
if key in new_infos:
infos[key].extend(new_infos[key])
@@ -214,5 +314,28 @@
attrs['requirements'] = sys.argv[1]
get_infos(**attrs)
+
+def install(project):
+ logger.info('Getting information about "%s".' % project)
+ try:
+ info = get_infos(project)
+ except InstallationException:
+ logger.info('Cound not find "%s".' % project)
+ return
+
+ if info['install'] == []:
+ logger.info('Nothing to install.')
+ return
+
+ install_path = get_config_var('base')
+ try:
+ install_from_infos(info['install'], info['remove'], info['conflict'],
+ install_path=install_path)
+
+ except InstallationConflict, e:
+ projects = ['%s %s' % (p.name, p.metadata.version) for p in e.args[0]]
+ logger.info('"%s" conflicts with "%s"' % (project, ','.join(projects)))
+
+
if __name__ == '__main__':
main()
diff --git a/distutils2/run.py b/distutils2/run.py
--- a/distutils2/run.py
+++ b/distutils2/run.py
@@ -1,7 +1,9 @@
import os
import sys
from optparse import OptionParser
+import logging
+from distutils2 import logger
from distutils2.util import grok_environment_error
from distutils2.errors import (DistutilsSetupError, DistutilsArgError,
DistutilsError, CCompilerError)
@@ -9,6 +11,7 @@
from distutils2 import __version__
from distutils2._backport.pkgutil import get_distributions, get_distribution
from distutils2.depgraph import generate_graph
+from distutils2.install import install
# This is a barebones help message generated displayed when the user
# runs the setup script with no arguments at all. More useful help
@@ -114,8 +117,17 @@
return dist
+def _set_logger():
+ logger.setLevel(logging.INFO)
+ sth = logging.StreamHandler(sys.stderr)
+ sth.setLevel(logging.INFO)
+ logger.addHandler(sth)
+ logger.propagate = 0
+
+
def main():
"""Main entry point for Distutils2"""
+ _set_logger()
parser = OptionParser()
parser.disable_interspersed_args()
parser.usage = '%prog [options] cmd1 cmd2 ..'
@@ -136,6 +148,14 @@
action="store_true", dest="fgraph", default=False,
help="Display the full graph for installed distributions.")
+ parser.add_option("-i", "--install",
+ action="store", dest="install",
+ help="Install a project.")
+
+ parser.add_option("-r", "--remove",
+ action="store", dest="remove",
+ help="Remove a project.")
+
options, args = parser.parse_args()
if options.version:
print('Distutils2 %s' % __version__)
@@ -169,6 +189,10 @@
print(graph)
sys.exit(0)
+ if options.install is not None:
+ install(options.install)
+ sys.exit(0)
+
if len(args) == 0:
parser.print_help()
sys.exit(0)
diff --git a/distutils2/util.py b/distutils2/util.py
--- a/distutils2/util.py
+++ b/distutils2/util.py
@@ -674,83 +674,6 @@
return base, ext
-def unzip_file(filename, location, flatten=True):
- """Unzip the file (zip file located at filename) to the destination
- location"""
- if not os.path.exists(location):
- os.makedirs(location)
- zipfp = open(filename, 'rb')
- try:
- zip = zipfile.ZipFile(zipfp)
- leading = has_leading_dir(zip.namelist()) and flatten
- for name in zip.namelist():
- data = zip.read(name)
- fn = name
- if leading:
- fn = split_leading_dir(name)[1]
- fn = os.path.join(location, fn)
- dir = os.path.dirname(fn)
- if not os.path.exists(dir):
- os.makedirs(dir)
- if fn.endswith('/') or fn.endswith('\\'):
- # A directory
- if not os.path.exists(fn):
- os.makedirs(fn)
- else:
- fp = open(fn, 'wb')
- try:
- fp.write(data)
- finally:
- fp.close()
- finally:
- zipfp.close()
-
-
-def untar_file(filename, location):
- """Untar the file (tar file located at filename) to the destination
- location
- """
- if not os.path.exists(location):
- os.makedirs(location)
- if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'):
- mode = 'r:gz'
- elif (filename.lower().endswith('.bz2')
- or filename.lower().endswith('.tbz')):
- mode = 'r:bz2'
- elif filename.lower().endswith('.tar'):
- mode = 'r'
- else:
- mode = 'r:*'
- tar = tarfile.open(filename, mode)
- try:
- leading = has_leading_dir([member.name for member in tar.getmembers()])
- for member in tar.getmembers():
- fn = member.name
- if leading:
- fn = split_leading_dir(fn)[1]
- path = os.path.join(location, fn)
- if member.isdir():
- if not os.path.exists(path):
- os.makedirs(path)
- else:
- try:
- fp = tar.extractfile(member)
- except (KeyError, AttributeError):
- # Some corrupt tar files seem to produce this
- # (specifically bad symlinks)
- continue
- if not os.path.exists(os.path.dirname(path)):
- os.makedirs(os.path.dirname(path))
- destfp = open(path, 'wb')
- try:
- shutil.copyfileobj(fp, destfp)
- finally:
- destfp.close()
- fp.close()
- finally:
- tar.close()
-
-
def has_leading_dir(paths):
"""Returns true if all the paths have the same leading path name
(i.e., everything is in one subdirectory in an archive)"""
--
Repository URL: http://hg.python.org/distutils2
More information about the Python-checkins
mailing list