Author: armin.ronacher Date: Thu Jun 26 13:11:20 2008 New Revision: 64530 Log: Implemented JSONHTMLBuilder and improved JSON handling (it now prefers json from the 2.6 stdlib or simplejson). Added: doctools/trunk/sphinx/util/_json.py Modified: doctools/trunk/CHANGES doctools/trunk/doc/builders.rst doctools/trunk/sphinx/builder.py doctools/trunk/sphinx/quickstart.py doctools/trunk/sphinx/search.py doctools/trunk/sphinx/templates/genindex-single.html doctools/trunk/sphinx/templates/genindex.html doctools/trunk/sphinx/util/json.py Modified: doctools/trunk/CHANGES ============================================================================== --- doctools/trunk/CHANGES (original) +++ doctools/trunk/CHANGES Thu Jun 26 13:11:20 2008 @@ -9,6 +9,9 @@ `PickleHTMLBuilder` is a concrete subclass of it that uses pickle as serialization implementation. +* `JSONHTMLBuilder` was added that similarily to `PickleHTMLBuilder` + dumps the generated HTML into JSON files for further processing. + Release 0.4 (Jun 23, 2008) ========================== Modified: doctools/trunk/doc/builders.rst ============================================================================== --- doctools/trunk/doc/builders.rst (original) +++ doctools/trunk/doc/builders.rst Thu Jun 26 13:11:20 2008 @@ -34,8 +34,7 @@ This builder produces a directory with pickle files containing mostly HTML fragments and TOC information, for use of a web application (or custom - postprocessing tool) that doesn't use the standard HTML templates. It also - is the format used by the Sphinx Web application. + postprocessing tool) that doesn't use the standard HTML templates. See :ref:`serialization-details` for details about the output format. @@ -44,6 +43,21 @@ The file suffix is ``.fpickle``. The global context is called ``globalcontext.pickle``, the search index ``searchindex.pickle``. +.. class:: JSONHTMLBuilder + + This builder produces a directory with JSON files containing mostly HTML + fragments and TOC information, for use of a web application (or custom + postprocessing tool) that doesn't use the standard HTML templates. + + See :ref:`serialization-details` for details about the output format. + + Its name is ``json``. + + The file suffix is ``.fjson``. The global context is called + ``globalcontext.json``, the search index ``searchindex.json``. + + .. versionadded:: 0.5 + .. class:: LaTeXBuilder This builder produces a bunch of LaTeX files in the output directory. You @@ -70,17 +84,19 @@ (`pickle`, `simplejson`, `phpserialize`, and others) to dump the generated HTML documentation. The pickle builder is a subclass of it. - A concreate subclass of this builder serializing to JSON could look like - this:: + A concreate subclass of this builder serializing to the `PHP serialization`_ + format could look like this:: + + import phpserialize - import simplejson + classs PHPSerializedBuilder(SerializingHTMLBuilder): + name = 'phpserialized' + implementation = phpserialize + out_suffix = '.file.phpdump' + globalcontext_filename = 'globalcontext.phpdump' + searchindex_filename = 'searchindex.phpdump' - classs JSONBuilder(SerializingHTMLBuilder): - name = 'json' - implementation = simplejson - out_suffix = '.fjson' - globalcontext_filename = 'globalcontext.json' - searchindex_filename = 'searchindex.json' + .. _PHP serialization: http://pypi.python.org/pypi/phpserialize .. attribute:: implementation Modified: doctools/trunk/sphinx/builder.py ============================================================================== --- doctools/trunk/sphinx/builder.py (original) +++ doctools/trunk/sphinx/builder.py Thu Jun 26 13:11:20 2008 @@ -25,7 +25,7 @@ from docutils.readers.doctree import Reader as DoctreeReader from sphinx import addnodes -from sphinx.util import ensuredir, relative_uri, SEP, os_path +from sphinx.util import ensuredir, relative_uri, SEP, os_path, json from sphinx.htmlhelp import build_hhx from sphinx.htmlwriter import HTMLWriter, HTMLTranslator, SmartyPantsHTMLTranslator from sphinx.textwriter import TextWriter @@ -294,7 +294,7 @@ name = 'html' copysource = True out_suffix = '.html' - indexer_format = 'json' + indexer_format = json supported_image_types = ['image/svg+xml', 'image/png', 'image/gif', 'image/jpeg'] searchindex_filename = 'searchindex.json' @@ -379,8 +379,7 @@ builder = self.name, parents = [], logo = logo, - favicon = favicon, - len = len, # the built-in + favicon = favicon ) def get_doc_context(self, docname, body): @@ -624,12 +623,14 @@ def load_indexer(self, docnames): try: - f = open(path.join(self.outdir, self.searchindex_filename), 'r') + f = open(path.join(self.outdir, self.searchindex_filename), 'rb') try: self.indexer.load(f, self.indexer_format) finally: f.close() - except (IOError, OSError): + except (IOError, OSError, NotImplementedError): + # we catch NotImplementedError here because if no simplejson + # is installed the searchindex can't be loaded pass # delete all entries for files that will be rebuilt self.indexer.prune(set(self.env.all_docs) - set(docnames)) @@ -683,9 +684,9 @@ def handle_finish(self): self.info(bold('dumping search index...')) self.indexer.prune(self.env.all_docs) - f = open(path.join(self.outdir, 'searchindex.json'), 'w') + f = open(path.join(self.outdir, self.searchindex_filename), 'wb') try: - self.indexer.dump(f, 'json') + self.indexer.dump(f, self.indexer_format) finally: f.close() @@ -702,9 +703,6 @@ #: the filename for the global context file globalcontext_filename = None - #: If set to `None` the indexer uses the serialization implementation - indexer_format = None - supported_image_types = ('image/svg+xml', 'image/png', 'image/gif', 'image/jpeg') @@ -712,6 +710,9 @@ self.init_translator_class() self.templates = None # no template bridge necessary + indexer_format = property(lambda x: x.implementation, doc=''' + Alias the indexer format to the serilization implementation''') + def get_target_uri(self, docname, typ=None): if docname == 'index': return '' @@ -754,13 +755,8 @@ finally: f.close() - self.info(bold('dumping search index...')) - self.indexer.prune(self.env.all_docs) - f = open(path.join(self.outdir, self.searchindex_filename), 'wb') - try: - self.indexer.dump(f, self.indexer_format or self.implementation) - finally: - f.close() + # super here to dump the search index + StandaloneHTMLBuilder.handle_finish(self) # copy the environment file from the doctree dir to the output dir # as needed by the web app @@ -773,6 +769,9 @@ class PickleHTMLBuilder(SerializingHTMLBuilder): + """ + A Builder that dumps the generated HTML into pickle files. + """ implementation = pickle name = 'pickle' out_suffix = '.fpickle' @@ -780,6 +779,17 @@ searchindex_filename = 'searchindex.pickle' +class JSONHTMLBuilder(SerializingHTMLBuilder): + """ + A builder that dumps the generated HTML into JSON files. + """ + implementation = json + name = 'json' + out_suffix = '.fjson' + globalcontext_filename = 'globalcontext.json' + searchindex_filename = 'searchindex.json' + + class HTMLHelpBuilder(StandaloneHTMLBuilder): """ Builder that also outputs Windows HTML help project, contents and index files. @@ -1128,6 +1138,7 @@ builtin_builders = { 'html': StandaloneHTMLBuilder, 'pickle': PickleHTMLBuilder, + 'json': JSONHTMLBuilder, 'web': PickleHTMLBuilder, 'htmlhelp': HTMLHelpBuilder, 'latex': LaTeXBuilder, Modified: doctools/trunk/sphinx/quickstart.py ============================================================================== --- doctools/trunk/sphinx/quickstart.py (original) +++ doctools/trunk/sphinx/quickstart.py Thu Jun 26 13:11:20 2008 @@ -238,7 +238,8 @@ help: \t@echo "Please use \\`make <target>' where <target> is one of" \t@echo " html to make standalone HTML files" -\t@echo " pickle to make pickle files (usable by e.g. sphinx-web)" +\t@echo " pickle to make pickle files" +\t@echo " json to make JSON files" \t@echo " htmlhelp to make HTML files and a HTML help project" \t@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" \t@echo " changes to make an overview over all changed/added/deprecated items" @@ -261,6 +262,12 @@ web: pickle +json: +\tmkdir -p %(rbuilddir)s/json %(rbuilddir)s/doctrees +\t$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) %(rbuilddir)s/json +\t@echo +\t@echo "Build finished; now you can process the JSON files." + htmlhelp: \tmkdir -p %(rbuilddir)s/htmlhelp %(rbuilddir)s/doctrees \t$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) %(rbuilddir)s/htmlhelp Modified: doctools/trunk/sphinx/search.py ============================================================================== --- doctools/trunk/sphinx/search.py (original) +++ doctools/trunk/sphinx/search.py Thu Jun 26 13:11:20 2008 @@ -9,7 +9,7 @@ :license: BSD. """ import re -import pickle +import cPickle as pickle from docutils.nodes import Text, NodeVisitor Modified: doctools/trunk/sphinx/templates/genindex-single.html ============================================================================== --- doctools/trunk/sphinx/templates/genindex-single.html (original) +++ doctools/trunk/sphinx/templates/genindex-single.html Thu Jun 26 13:11:20 2008 @@ -24,7 +24,7 @@ {%- endfor %} </dl></dd> {%- endif -%} -{%- set numitems = numitems + 1 + len(subitems) -%} +{%- set numitems = numitems + 1 + subitems|length -%} {%- if numcols < 2 and numitems > breakat -%} {%- set numcols = numcols+1 -%} </dl></td><td width="33%" valign="top"><dl> Modified: doctools/trunk/sphinx/templates/genindex.html ============================================================================== --- doctools/trunk/sphinx/templates/genindex.html (original) +++ doctools/trunk/sphinx/templates/genindex.html Thu Jun 26 13:11:20 2008 @@ -32,7 +32,7 @@ {%- endfor %} </dl></dd> {%- endif -%} -{%- set numitems = numitems + 1 + len(subitems) -%} +{%- set numitems = numitems + 1 + subitems|length -%} {%- if numcols < 2 and numitems > breakat -%} {%- set numcols = numcols+1 -%} </dl></td><td width="33%" valign="top"><dl> Added: doctools/trunk/sphinx/util/_json.py ============================================================================== --- (empty file) +++ doctools/trunk/sphinx/util/_json.py Thu Jun 26 13:11:20 2008 @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- +""" + sphinx.util._json + ~~~~~~~~~~~~~~~~~ + + This module implements a simple JSON serializer if simplejson is + unavailable. + + This is not fully JSON compliant but enough for the searchindex. + And the generated files are smaller than the simplejson ones. + + Uses the basestring encode function from simplejson. + + :copyright: Copyright 2008 by Armin Ronacher, Bob Ippolito. + :license: BSD. +""" +import re + + +# escape \, ", control characters and everything outside ASCII +ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') +ESCAPE_DICT = { + '\\': '\\\\', + '"': '\\"', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', +} + + +def encode_basestring_ascii(s): + def replace(match): + s = match.group(0) + try: + return ESCAPE_DICT[s] + except KeyError: + n = ord(s) + if n < 0x10000: + return '\\u%04x' % (n,) + else: + # surrogate pair + n -= 0x10000 + s1 = 0xd800 | ((n >> 10) & 0x3ff) + s2 = 0xdc00 | (n & 0x3ff) + return '\\u%04x\\u%04x' % (s1, s2) + return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' + + +def dumps(obj, key=False): + if key: + if not isinstance(obj, basestring): + obj = str(obj) + return encode_basestring_ascii(obj) + if obj is None: + return 'null' + elif obj is True or obj is False: + return obj and 'true' or 'false' + elif isinstance(obj, (int, long, float)): + return str(obj) + elif isinstance(obj, dict): + return '{%s}' % ','.join('%s:%s' % ( + dumps(key, True), + dumps(value) + ) for key, value in obj.iteritems()) + elif isinstance(obj, (tuple, list, set)): + return '[%s]' % ','.join(dumps(x) for x in obj) + elif isinstance(obj, basestring): + return encode_basestring_ascii(obj) + raise TypeError(type(obj)) + + +def dump(obj, f): + f.write(dumps(obj)) Modified: doctools/trunk/sphinx/util/json.py ============================================================================== --- doctools/trunk/sphinx/util/json.py (original) +++ doctools/trunk/sphinx/util/json.py Thu Jun 26 13:11:20 2008 @@ -3,87 +3,32 @@ sphinx.util.json ~~~~~~~~~~~~~~~~ - Minimal JSON module that generates small dumps. + This module imports JSON functions from various locations. - This is not fully JSON compliant but enough for the searchindex. - And the generated files are smaller than the simplejson ones. - - Uses the basestring encode function from simplejson. - - :copyright: 2007-2008 by Armin Ronacher, Bob Ippolito. + :copyright: Copyright 2008 by Armin Ronacher. :license: BSD. """ -import re - -# escape \, ", control characters and everything outside ASCII -ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') -ESCAPE_DICT = { - '\\': '\\\\', - '"': '\\"', - '\b': '\\b', - '\f': '\\f', - '\n': '\\n', - '\r': '\\r', - '\t': '\\t', -} - - -def encode_basestring_ascii(s): - def replace(match): - s = match.group(0) - try: - return ESCAPE_DICT[s] - except KeyError: - n = ord(s) - if n < 0x10000: - return '\\u%04x' % (n,) - else: - # surrogate pair - n -= 0x10000 - s1 = 0xd800 | ((n >> 10) & 0x3ff) - s2 = 0xdc00 | (n & 0x3ff) - return '\\u%04x\\u%04x' % (s1, s2) - return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' - - -def dump_json(obj, key=False): - if key: - if not isinstance(obj, basestring): - obj = str(obj) - return encode_basestring_ascii(obj) - if obj is None: - return 'null' - elif obj is True or obj is False: - return obj and 'true' or 'false' - elif isinstance(obj, (int, long, float)): - return str(obj) - elif isinstance(obj, dict): - return '{%s}' % ','.join('%s:%s' % ( - dump_json(key, True), - dump_json(value) - ) for key, value in obj.iteritems()) - elif isinstance(obj, (tuple, list, set)): - return '[%s]' % ','.join(dump_json(x) for x in obj) - elif isinstance(obj, basestring): - return encode_basestring_ascii(obj) - raise TypeError(type(obj)) - - -STRING = re.compile(r'("(\\\\|\\"|[^"])*")') - -def load_json(s): - d = {'null': None, 'true': True, 'false': False} - s = STRING.sub(r'u\1', s) - return eval(s, d) - - -# serializer interface -dumps = dump_json -loads = load_json +# if no simplejson is available this module can not load json files. +can_load = True -def dump(obj, f): - f.write(dumps(obj)) +# unset __name__ for a moment so that the import goes straight into +# the stdlib for python 2.4. +_old_name = __name__ +del __name__ + +try: + from simplejson import dumps, dump, loads, load +except ImportError: + try: + from json import dumps, dump, loads, load + except ImportError: + from sphinx.util._json import dumps, dump + def _dummy(x): + raise NotImplementedError('simplejson unavailable, can\'t load') + load = loads = _dummy + can_load = False + del _dummy -def load(f): - return loads(f.read()) +__name__ = _old_name +del _old_name