Author: armin.ronacher Date: Thu Jun 26 11:40:42 2008 New Revision: 64529 Log: Refactored pickle builder into a SerializingHTMLBuilder and PickleHTMLBuilder. Subclasses can change the serialization format easily. Modified: doctools/trunk/CHANGES doctools/trunk/doc/builders.rst doctools/trunk/sphinx/builder.py doctools/trunk/sphinx/search.py doctools/trunk/sphinx/util/json.py Modified: doctools/trunk/CHANGES ============================================================================== --- doctools/trunk/CHANGES (original) +++ doctools/trunk/CHANGES Thu Jun 26 11:40:42 2008 @@ -1,3 +1,15 @@ +Release 0.5 (in development) +============================ + +New features added +------------------ + +* `SerializingHTMLBuilder` was added as new abstract builder that can + be subclassed to serialize build HTML in a specific format. The + `PickleHTMLBuilder` is a concrete subclass of it that uses pickle as + serialization implementation. + + Release 0.4 (Jun 23, 2008) ========================== Modified: doctools/trunk/doc/builders.rst ============================================================================== --- doctools/trunk/doc/builders.rst (original) +++ doctools/trunk/doc/builders.rst Thu Jun 26 11:40:42 2008 @@ -37,10 +37,13 @@ postprocessing tool) that doesn't use the standard HTML templates. It also is the format used by the Sphinx Web application. - See :ref:`pickle-details` for details about the output format. + See :ref:`serialization-details` for details about the output format. Its name is ``pickle``. (The old name ``web`` still works as well.) + The file suffix is ``.fpickle``. The global context is called + ``globalcontext.pickle``, the search index ``searchindex.pickle``. + .. class:: LaTeXBuilder This builder produces a bunch of LaTeX files in the output directory. You @@ -60,6 +63,51 @@ Its name is ``text``. .. versionadded:: 0.4 + +.. class:: SerializingHTMLBuilder + + This builder uses a module that implements the Python serialization API + (`pickle`, `simplejson`, `phpserialize`, and others) to dump the generated + HTML documentation. The pickle builder is a subclass of it. + + A concreate subclass of this builder serializing to JSON could look like + this:: + + import simplejson + + classs JSONBuilder(SerializingHTMLBuilder): + name = 'json' + implementation = simplejson + out_suffix = '.fjson' + globalcontext_filename = 'globalcontext.json' + searchindex_filename = 'searchindex.json' + + .. attribute:: implementation + + A module that implements `dump()`, `load()`, `dumps()` and `loads()` + functions that conform to the functions with the same names from the + pickle module. Known modules implementing this interface are + `simplejson` (or `json` in Python 2.6), `phpserialize`, `plistlib`, + and others. + + .. attribute:: out_suffix + + The suffix for all regular files. + + .. attribute:: globalcontext_filename + + The filename for the file that contains the "global context". This + is a dict with some general configuration values such as the name + of the project. + + .. attribute:: searchindex_filename + + The filename for the search index Sphinx generates. + + + See :ref:`serialization-details` for details about the output format. + + .. versionadded:: 0.5 .. class:: ChangesBuilder @@ -85,18 +133,22 @@ * :mod:`~sphinx.ext.coverage` -.. _pickle-details: +.. _serialization-details: -Pickle builder details ----------------------- +Serialization builder details +----------------------------- -The builder outputs one pickle file per source file, and a few special files. -It also copies the reST source files in the directory ``_sources`` under the -output directory. - -The files per source file have the extensions ``.fpickle``, and are arranged in -directories just as the source files are. They unpickle to a dictionary with -these keys: +All serialization builders outputs one file per source file and a few special +files. They also copy the reST source files in the directory ``_sources`` +under the output directory. + +The :class:`PickleHTMLBuilder` is a builtin subclass that implements the pickle +serialization interface. + +The files per source file have the extensions of +:attr:`~SerializingHTMLBuilder.out_suffix`, and are arranged in directories +just as the source files are. They unserialize to a dictionary (or dictionary +like structure) with these keys: ``body`` The HTML "body" (that is, the HTML rendering of the source file), as rendered @@ -125,10 +177,7 @@ The special files are located in the root output directory. They are: -``environment.pickle`` - The build environment. (XXX add important environment properties) - -``globalcontext.pickle`` +:attr:`SerializingHTMLBuilder.globalcontext_filename` A pickled dict with these keys: ``project``, ``copyright``, ``release``, ``version`` @@ -147,7 +196,7 @@ ``titles`` A dictionary of all documents' titles, as HTML strings. -``searchindex.pickle`` +:attr:`SerializingHTMLBuilder.searchindex_filename` An index that can be used for searching the documentation. It is a pickled list with these entries: @@ -156,3 +205,11 @@ list. * A dict mapping word roots (processed by an English-language stemmer) to a list of integers, which are indices into the first list. + +``environment.pickle`` + The build environment. This is always a pickle file, independent of the + builder and a copy of the environment that was used when the builder was + started. (XXX: document common members) + + Unlike the other pickle files this pickle file requires that the sphinx + module is available on unpickling. Modified: doctools/trunk/sphinx/builder.py ============================================================================== --- doctools/trunk/sphinx/builder.py (original) +++ doctools/trunk/sphinx/builder.py Thu Jun 26 11:40:42 2008 @@ -297,6 +297,7 @@ indexer_format = 'json' supported_image_types = ['image/svg+xml', 'image/png', 'image/gif', 'image/jpeg'] + searchindex_filename = 'searchindex.json' def init(self): """Load templates.""" @@ -623,7 +624,7 @@ def load_indexer(self, docnames): try: - f = open(path.join(self.outdir, 'searchindex.'+self.indexer_format), 'r') + f = open(path.join(self.outdir, self.searchindex_filename), 'r') try: self.indexer.load(f, self.indexer_format) finally: @@ -638,7 +639,7 @@ if self.indexer is not None and title: self.indexer.feed(pagename, title, doctree) - # --------- these are overwritten by the Pickle builder + # --------- these are overwritten by the serialization builder def get_target_uri(self, docname, typ=None): return docname + self.out_suffix @@ -689,13 +690,21 @@ f.close() -class PickleHTMLBuilder(StandaloneHTMLBuilder): +class SerializingHTMLBuilder(StandaloneHTMLBuilder): """ - Builds HTML docs without rendering templates. + An abstract builder that serializes the HTML generated. """ - name = 'pickle' - out_suffix = '.fpickle' - indexer_format = 'pickle' + #: the serializing implementation to use. Set this to a module that + #: implements a `dump`, `load`, `dumps` and `loads` functions + #: (pickle, simplejson etc.) + implementation = None + + #: the filename for the global context file + globalcontext_filename = None + + #: If set to `None` the indexer uses the serialization implementation + indexer_format = None + supported_image_types = ('image/svg+xml', 'image/png', 'image/gif', 'image/jpeg') @@ -724,7 +733,7 @@ ensuredir(path.dirname(outfilename)) f = open(outfilename, 'wb') try: - pickle.dump(ctx, f, 2) + self.implementation.dump(ctx, f, 2) finally: f.close() @@ -738,18 +747,18 @@ def handle_finish(self): # dump the global context - outfilename = path.join(self.outdir, 'globalcontext.pickle') + outfilename = path.join(self.outdir, self.globalcontext_filename) f = open(outfilename, 'wb') try: - pickle.dump(self.globalcontext, f, 2) + self.implementation.dump(self.globalcontext, f, 2) finally: f.close() self.info(bold('dumping search index...')) self.indexer.prune(self.env.all_docs) - f = open(path.join(self.outdir, 'searchindex.pickle'), 'wb') + f = open(path.join(self.outdir, self.searchindex_filename), 'wb') try: - self.indexer.dump(f, 'pickle') + self.indexer.dump(f, self.indexer_format or self.implementation) finally: f.close() @@ -763,6 +772,14 @@ open(path.join(self.outdir, LAST_BUILD_FILENAME), 'w').close() +class PickleHTMLBuilder(SerializingHTMLBuilder): + implementation = pickle + name = 'pickle' + out_suffix = '.fpickle' + globalcontext_filename = 'globalcontext.pickle' + searchindex_filename = 'searchindex.pickle' + + class HTMLHelpBuilder(StandaloneHTMLBuilder): """ Builder that also outputs Windows HTML help project, contents and index files. Modified: doctools/trunk/sphinx/search.py ============================================================================== --- doctools/trunk/sphinx/search.py (original) +++ doctools/trunk/sphinx/search.py Thu Jun 26 11:40:42 2008 @@ -14,7 +14,7 @@ from docutils.nodes import Text, NodeVisitor from sphinx.util.stemmer import PorterStemmer -from sphinx.util.json import dump_json, load_json +from sphinx.util import json word_re = re.compile(r'\w+(?u)') @@ -50,8 +50,8 @@ passed to the `feed` method. """ formats = { - 'json': (dump_json, load_json), - 'pickle': (pickle.dumps, pickle.loads), + 'json': json, + 'pickle': pickle } def __init__(self): @@ -63,7 +63,9 @@ def load(self, stream, format): """Reconstruct from frozen data.""" - frozen = self.formats[format][1](stream.read()) + if isinstance(format, basestring): + format = self.formats[format] + frozen = format.load(stream) index2fn = frozen[0] self._titles = dict(zip(frozen[0], frozen[1])) self._mapping = dict((k, set(index2fn[i] for i in v)) @@ -71,7 +73,9 @@ def dump(self, stream, format): """Dump the frozen index to a stream.""" - stream.write(self.formats[format][0](self.freeze())) + if isinstance(format, basestring): + format = self.formats[format] + format.dump(self.freeze(), stream) def freeze(self): """ Modified: doctools/trunk/sphinx/util/json.py ============================================================================== --- doctools/trunk/sphinx/util/json.py (original) +++ doctools/trunk/sphinx/util/json.py Thu Jun 26 11:40:42 2008 @@ -76,3 +76,14 @@ d = {'null': None, 'true': True, 'false': False} s = STRING.sub(r'u\1', s) return eval(s, d) + + +# serializer interface +dumps = dump_json +loads = load_json + +def dump(obj, f): + f.write(dumps(obj)) + +def load(f): + return loads(f.read())