[XML-SIG] pyxml minidom: I can remove and append, but not replace
Joseph Reagle
reagle@mit.edu
Mon, 21 Apr 2003 11:36:06 -0400
--Boundary-00=_m/Ap+NmbP9tRrN5
Content-Type: text/plain;
charset="us-ascii"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline
I have a snippet that replacing a title element with a new element with a
modified one (base on an amazon query) -- if that title element already
exists. However, I can't do an insertBefore or replace for some odd reason:
for property in book_dtd: # step through the DTD
feature = _get_feature(property,book)
if property == "title":
r_title = doc.createElementNS(book_ns,"title")
r_title.appendChild(doc.createTextNode(title))
if feature:
book.appendChild(r_title)
book.removeChild(feature)
# book.replaceChild(feature,r_title)
else:
book.appendChild(r_title)
If I were to comment the "book.{append,remove}Child" and uncomment the
book.replaceChild I get the following error:
Traceback (most recent call last):
File "./pybook.py", line 163, in ?
bookAugment(doc)
File "./pybook.py", line 100, in bookAugment
book.replaceChild(feature,r_title)
File "/usr/lib/python2.2/site-packages/_xmlplus/dom/minidom.py", line 145,
in replaceChild
raise xml.dom.NotFoundErr()
xml.dom.NotFoundErr: Node does not exist in this context
The node is using the same namespace, and I've used the replaceChild in
other cases (where I'm actually using a node from a different XML
instance!) .py and test file attached
--Boundary-00=_m/Ap+NmbP9tRrN5
Content-Type: text/x-python;
charset="us-ascii";
name="amazon.py"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="amazon.py"
"""Python wrapper
for Amazon web APIs
This module allows you to access Amazon's web APIs,
to do things like search Amazon and get the results programmatically.
Described here:
http://www.amazon.com/webservices
You need a Amazon-provided license key to use these services.
Follow the link above to get one. These functions will look in
several places (in this order) for the license key:
- the "license_key" argument of each function
- the module-level LICENSE_KEY variable (call setLicense once to set it)
- an environment variable called AMAZON_LICENSE_KEY
- a file called ".amazonkey" in the current directory
- a file called "amazonkey.txt" in the current directory
- a file called ".amazonkey" in your home directory
- a file called "amazonkey.txt" in your home directory
- a file called ".amazonkey" in the same directory as amazon.py
- a file called "amazonkey.txt" in the same directory as amazon.py
Sample usage:
>>> import amazon
>>> amazon.setLicense('...') # must get your own key!
>>> pythonBooks = amazon.searchByKeyword('Python')
>>> pythonBooks[0].ProductName
u'Learning Python (Help for Programmers)'
>>> pythonBooks[0].URL
...
>>> pythonBooks[0].OurPrice
...
Other available functions:
- browseBestSellers
- searchByASIN
- searchByUPC
- searchByAuthor
- searchByArtist
- searchByActor
- searchByDirector
- searchByManufacturer
- searchByListMania
- searchSimilar
- searchByWishlist
Other usage notes:
- Most functions can take product_line as well, see source for possible values
- All functions can take type="lite" to get less detail in results
- All functions can take page=N to get second, third, fourth page of results
- All functions can take license_key="XYZ", instead of setting it globally
- All functions can take http_proxy="http://x/y/z" which overrides your system setting
"""
__author__ = "Mark Pilgrim (f8dy@diveintomark.org)"
__version__ = "0.5"
__cvsversion__ = "$Revision: 1.6 $"[11:-2]
__date__ = "$Date: 2003/04/04 03:08:34 $"[7:-2]
__copyright__ = "Copyright (c) 2002 Mark Pilgrim"
__license__ = "Python"
# Powersearch and return object type fix by Joseph Reagle <geek@goatee.net>
from xml.dom import minidom
import os, sys, getopt, cgi, urllib
try:
import timeoutsocket # http://www.timo-tasi.org/python/timeoutsocket.py
timeoutsocket.setDefaultSocketTimeout(10)
except ImportError:
pass
LICENSE_KEY = None
HTTP_PROXY = None
# don't touch the rest of these constants
class AmazonError(Exception): pass
class NoLicenseKey(Exception): pass
_amazonfile1 = ".amazonkey"
_amazonfile2 = "amazonkey.txt"
_licenseLocations = (
(lambda key: key, 'passed to the function in license_key variable'),
(lambda key: LICENSE_KEY, 'module-level LICENSE_KEY variable (call setLicense to set it)'),
(lambda key: os.environ.get('AMAZON_LICENSE_KEY', None), 'an environment variable called AMAZON_LICENSE_KEY'),
(lambda key: _contentsOf(os.getcwd(), _amazonfile1), '%s in the current directory' % _amazonfile1),
(lambda key: _contentsOf(os.getcwd(), _amazonfile2), '%s in the current directory' % _amazonfile2),
(lambda key: _contentsOf(os.environ.get('HOME', ''), _amazonfile1), '%s in your home directory' % _amazonfile1),
(lambda key: _contentsOf(os.environ.get('HOME', ''), _amazonfile2), '%s in your home directory' % _amazonfile2),
(lambda key: _contentsOf(_getScriptDir(), _amazonfile1), '%s in the amazon.py directory' % _amazonfile1),
(lambda key: _contentsOf(_getScriptDir(), _amazonfile2), '%s in the amazon.py directory' % _amazonfile2)
)
## administrative functions
def version():
print """PyAmazon %(__version__)s
%(__copyright__)s
released %(__date__)s
""" % globals()
## utility functions
def setLicense(license_key):
"""set license key"""
global LICENSE_KEY
LICENSE_KEY = license_key
def getLicense(license_key = None):
"""get license key
license key can come from any number of locations;
see module docs for search order"""
for get, location in _licenseLocations:
rc = get(license_key)
if rc: return rc
raise NoLicenseKey, 'get a license key at http://www.amazon.com/webservices'
def setProxy(http_proxy):
"""set HTTP proxy"""
global HTTP_PROXY
HTTP_PROXY = http_proxy
def getProxy(http_proxy = None):
"""get HTTP proxy"""
return http_proxy or HTTP_PROXY
def getProxies(http_proxy = None):
http_proxy = getProxy(http_proxy)
if http_proxy:
proxies = {"http": http_proxy}
else:
proxies = None
return proxies
def _contentsOf(dirname, filename):
filename = os.path.join(dirname, filename)
if not os.path.exists(filename): return None
fsock = open(filename)
contents = fsock.read()
fsock.close()
return contents
def _getScriptDir():
if __name__ == '__main__':
return os.path.abspath(os.path.dirname(sys.argv[0]))
else:
return os.path.abspath(os.path.dirname(sys.modules[__name__].__file__))
class Bag: pass
def unmarshal(element):
rc = Bag()
if isinstance(element, minidom.Element) and (element.tagName == 'Details'):
rc.URL = element.attributes["url"].value
childElements = [e for e in element.childNodes if isinstance(e, minidom.Element)]
if childElements:
for child in childElements:
key = child.tagName
if hasattr(rc, key):
if type(getattr(rc, key)) <> type([]):
setattr(rc, key, [getattr(rc, key)])
setattr(rc, key, getattr(rc, key) + [unmarshal(child)])
elif child.tagName in ['Details']:
# make the first Details element a list
setattr(rc,key,[unmarshal(child)])
#dbg: because otherwise 'hasattr' only tests
#dbg: on the second occurence: if there's a
#dbg: single return to a query, it's not a
#dbg: list. This module should always
#dbg: return a list of Details objects.
else:
setattr(rc, key, unmarshal(child))
else:
rc = "".join([e.data for e in element.childNodes if isinstance(e, minidom.Text)])
if element.tagName == 'SalesRank':
rc = int(rc.replace(',', ''))
return rc
def buildURL(search_type, keyword, product_line, type, page, license_key):
url = "http://xml.amazon.com/onca/xml?v=1.0&f=xml&t=webservices-20"
url += "&dev-t=%s" % license_key.strip()
url += "&type=%s" % type
if page:
url += "&page=%s" % page
if product_line:
url += "&mode=%s" % product_line
url += "&%s=%s" % (search_type, urllib.quote(keyword))
#dbg: title searches with a ":" yields no results
print url
return url
## main functions
def search(search_type, keyword, product_line, type="heavy", page=None,
license_key = None, http_proxy = None, return_xml = 0):
"""search Amazon
You need a license key to call this function; see
http://www.amazon.com/webservices
to get one. Then you can either pass it to
this function every time, or set it globally; see the module docs for details.
Parameters:
keyword - keyword to search
search_type - in (KeywordSearch, BrowseNodeSearch, AsinSearch, UpcSearch, AuthorSearch, ArtistSearch, ActorSearch, DirectorSearch, ManufacturerSearch, ListManiaSearch, SimilaritySearch)
product_line - type of product to search for. restrictions based on search_type
UpcSearch - in (music, classical)
AuthorSearch - must be "books"
ArtistSearch - in (music, classical)
ActorSearch - in (dvd, vhs, video)
DirectorSearch - in (dvd, vhs, video)
ManufacturerSearch - in (electronics, kitchen, videogames, software, photo, pc-hardware)
http_proxy (optional) - address of HTTP proxy to use for sending and receiving SOAP messages
Returns: list of Bags, each Bag may contain the following attributes:
Asin - Amazon ID ("ASIN" number) of this item
Authors - list of authors
Availability - "available", etc.
BrowseList - list of related categories
Catalog - catalog type ("Book", etc)
CollectiblePrice - ?, format "$34.95"
ImageUrlLarge - URL of large image of this item
ImageUrlMedium - URL of medium image of this item
ImageUrlSmall - URL of small image of this item
Isbn - ISBN number
ListPrice - list price, format "$34.95"
Lists - list of ListMania lists that include this item
Manufacturer - manufacturer
Media - media ("Paperback", "Audio CD", etc)
NumMedia - number of different media types in which this item is available
OurPrice - Amazon price, format "$24.47"
ProductName - name of this item
ReleaseDate - release date, format "09 April, 1999"
Reviews - reviews (AvgCustomerRating, plus list of CustomerReview with Rating, Summary, Content)
SalesRank - sales rank (integer)
SimilarProducts - list of Product, which is ASIN number
ThirdPartyNewPrice - ?, format "$34.95"
URL - URL of this item
"""
license_key = getLicense(license_key)
url = buildURL(search_type, keyword, product_line, type, page, license_key)
proxies = getProxies(http_proxy)
u = urllib.FancyURLopener(proxies)
usock = u.open(url)
xmldoc = minidom.parse(usock)
# from xml.dom.ext import PrettyPrint
# PrettyPrint(xmldoc)
usock.close()
if return_xml:
return xmldoc
else:
data = unmarshal(xmldoc).ProductInfo
if hasattr(data, 'ErrorMsg'):
raise AmazonError, data.ErrorMsg
else:
return data.Details
def searchByKeyword(keyword, product_line="books", type="heavy", page=1, license_key=None, http_proxy=None):
return search("KeywordSearch", keyword, product_line, type, page, license_key, http_proxy)
def browseBestSellers(browse_node, product_line="books", type="heavy", page=1, license_key=None, http_proxy=None):
return search("BrowseNodeSearch", browse_node, product_line, type, page, license_key, http_proxy)
def searchByASIN(ASIN, type="heavy", license_key=None, http_proxy=None):
return search("AsinSearch", ASIN, None, type, None, license_key, http_proxy)
def searchByUPC(UPC, type="heavy", license_key=None, http_proxy=None):
return search("UpcSearch", UPC, None, type, None, license_key, http_proxy)
def searchByAuthor(author, type="heavy", page=1, license_key=None, http_proxy=None):
return search("AuthorSearch", author, "books", type, page, license_key, http_proxy)
def searchByArtist(artist, product_line="music", type="heavy", page=1, license_key=None, http_proxy=None):
if product_line not in ("music", "classical"):
raise AmazonError, "product_line must be in ('music', 'classical')"
return search("ArtistSearch", artist, product_line, type, page, license_key, http_proxy)
def searchByActor(actor, product_line="dvd", type="heavy", page=1, license_key=None, http_proxy=None):
if product_line not in ("dvd", "vhs", "video"):
raise AmazonError, "product_line must be in ('dvd', 'vhs', 'video')"
return search("ActorSearch", actor, product_line, type, page, license_key, http_proxy)
def searchByDirector(director, product_line="dvd", type="heavy", page=1, license_key=None, http_proxy=None):
if product_line not in ("dvd", "vhs", "video"):
raise AmazonError, "product_line must be in ('dvd', 'vhs', 'video')"
return search("DirectorSearch", director, product_line, type, page, license_key, http_proxy)
def searchByManufacturer(manufacturer, product_line="pc-hardware", type="heavy", page=1, license_key=None, http_proxy=None):
if product_line not in ("electronics", "kitchen", "videogames", "software", "photo", "pc-hardware"):
raise AmazonError, "product_line must be in ('electronics', 'kitchen', 'videogames', 'software', 'photo', 'pc-hardware')"
return search("ManufacturerSearch", manufacturer, product_line, type, page, license_key, http_proxy)
def searchByListMania(listManiaID, type="heavy", page=1, license_key=None, http_proxy=None):
return search("ListManiaSearch", listManiaID, None, type, page, license_key, http_proxy)
def searchSimilar(ASIN, type="heavy", page=1, license_key=None, http_proxy=None):
return search("SimilaritySearch", ASIN, None, type, page, license_key, http_proxy)
def searchByWishlist(wishlistID, type="heavy", page=1, license_key=None, http_proxy=None):
return search("WishlistSearch", wishlistID, None, type, page, license_key, http_proxy)
def searchByPower(keyword, product_line="books", type="heavy", page=1, license_key=None, http_proxy=None, return_xml = 0):
return search("PowerSearch", keyword, product_line, type, page, license_key, http_proxy, return_xml)
# >>> RecentKing = amazon.searchByPower('author:Stephen King and pubdate:2003')
# >>> SnowCrash = amazon.searchByPower('title:Snow Crash')
--Boundary-00=_m/Ap+NmbP9tRrN5
Content-Type: text/x-python;
charset="us-ascii";
name="pybook.py"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="pybook.py"
#!/usr/bin/env python
# http://www.python.org/doc/current/lib/dom-node-objects.html
def bookAugment(doc):
import amazon, time
def getText(node):
node.normalize()
rc = node.firstChild.data
return _normalizeWhitespace(rc)
def replaceText(feature,value):
for node in feature.childNodes:
if node.nodeType == node.TEXT_NODE:
feature.removeChild(node)
print "***value is ", type(value), value
feature.appendChild(doc.createTextNode(value))
def buildQuery(query, property, value):
if value != "":
if query != "": query += " and "
query += "%s: %s" %(property, value)
return query
def _get_feature(property,book):
"""_get_feature returns a book childNode corresponding
to the property. This is extremely clumsy iterating
over the books children for every property"""
for feature in _childrenElements(book):
print "testing '%s' against '%s'" %(property,feature.localName)
if feature.localName == property:
return feature
def _splitTitle(title):
try:
title,subtitle = getText(ProductName).split(": ",1)
except ValueError:
title = getText(ProductName)
subtitle = ""
print "title = '%s', subtitle = '%s'" %(title, subtitle)
return title, subtitle
_childrenElements = lambda node: [n for n in node.childNodes
if n.nodeType == n.ELEMENT_NODE] # is node element
_normalizeWhitespace = lambda text: ' '.join(text.split())
_normalizeIsbn = lambda chars: chars.replace('-','')
query = "" # the query to pass to Amazon searchByPower
bookcase = doc.getElementsByTagName("bookcase").pop()
collection = doc.getElementsByTagName("collection").pop()
for book in _childrenElements(collection):
# Build the query from existing title, isbn, and author
for feature in _childrenElements(book):
if feature.localName == "title":
title = getText(feature)
query = buildQuery(query, "title", title)
if feature.localName == "authors":
for author in _childrenElements(feature):
author = getText(author)
query = buildQuery(query, "author", author)
if feature.localName == "isbn":
isbn = _normalizeIsbn(getText(feature))
query = buildQuery(query, "isbn", isbn)
# Perform the query
print "*** query = ", query.encode('utf-8')
try:
#results = amazon.searchByPower('author:Stephenson and title:Snow Crash')
results = amazon.searchByPower(query,return_xml=1)
except amazon.AmazonError, e:
print "*** ERRRRRROR", e
query = "" # Reset query
time.sleep(.7) # Amazon only permits one query per second
# Augment Book with results of query by iterating over
# the bookcase DTD and replacing/inserting elements
# dbg: I know this algorithm sucks.
Details = results.getElementsByTagName("Details")
if len(Details) == 1:
book_ns = "http://periapsis.org/bookcase/"
book_dtd = ("title", "subtitle", "authors", "isbn")
# book_dtd = ("title", "subtitle", "authors", "binding", "pur_date",
# "pur_price", "publisher", "edition", "cr_years", "pub_year",
# "isbn", "lccn", "pages", "languages", "genres", "keywords",
# "series", "series_num", "condition", "signed", "read", "gift",
# "loaned", "rating", "comments")
ProductName = results.getElementsByTagName("ProductName")[0]
print type(ProductName)
title, subtitle = _splitTitle(getText(ProductName).split(": ",1))
for property in book_dtd: # step through the DTD
feature = _get_feature(property,book)
if property == "title":
r_title = doc.createElementNS(book_ns,"title")
r_title.appendChild(doc.createTextNode(title))
if feature:
book.appendChild(r_title)
book.removeChild(feature)
# book.replaceChild(feature,r_title)
else:
book.appendChild(r_title)
elif property == "subtitle" and subtitle != "":
r_subtitle = doc.createElementNS(book_ns,"subtitle")
r_subtitle.appendChild(doc.createTextNode(subtitle))
if feature:
book.appendChild(r_subtitle)
book.removeChild(feature)
else:
book.appendChild(r_subtitle)
elif property == "authors":
# remove my children
for author in _childrenElements(feature):
feature.removeChild(author)
# add the children from the amazon result
for r_author in results.getElementsByTagName("Author"):
# lowercase the results to the bookcase convention
r_author.tagName = "author"
feature.appendChild(r_author)
elif property == "isbn":
r_isbn = results.getElementsByTagName("Isbn")[0]
r_isbn.tagName = "isbn"
if feature:
book.replaceChild(feature,r_isbn)
else:
book.appendChild(r_isbn)
PrettyPrint(bookcase)
def print_usage():
print "pybookcase infile.xml outfile.xml"
print "pybookcase will augment a bookcase XML file with other information from "
print "the python interface"
if __name__ == "__main__":
import getopt, sys
mode = 'xml'
try:
(options,files) = getopt.getopt (sys.argv[1:],"h")
except getopt.error:
print_usage()
for (option,value) in options:
pass
if option == '-h':
print_usage()
try:
infd = open(files[0])
except IndexError:
infd = sys.stdin
try:
outfd = open(files[1], 'w')
except IndexError:
outfd = sys.stdout
from xml.dom import minidom
from xml.dom.ext import PrettyPrint
doc = minidom.parse(infd)
bookAugment(doc)
infd.close()
outfd.close()
--Boundary-00=_m/Ap+NmbP9tRrN5
Content-Type: application/x-bookcase;
name="test.bc"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment; filename="test.bc"
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE bookcase SYSTEM "bookcase.dtd">
<bookcase xmlns="http://periapsis.org/bookcase/" syntaxVersion="2" >
<collection unitTitle="Books" title="Converted Books" unit="book" >
<book>
<title>The Diamond Age</title>
<subtitle>A Young Lady's Illustrated Primer</subtitle>
<authors>
<author>Neal Stephenson</author>
</authors>
<genres>
<genre>SciFi</genre>
</genres>
<read/>
</book>
<book>
<title>Vampire Book</title>
<subtitle></subtitle>
<authors>
<author>J. Gordon Melton</author>
</authors>
<genres>
<genre>History</genre>
</genres>
<read/>
</book>
<book>
<title>Road Warriors</title>
<subtitle>Dreams and Nightmares Along the Information Highway</subtitle>
<authors>
<author>Burstein</author>
<author>Kline</author>
</authors>
<genres>
<genre>History</genre>
</genres>
<read/>
</book>
</collection>
</bookcase>
--Boundary-00=_m/Ap+NmbP9tRrN5--