Python nuube needs Unicode help

gheissenberger at gheissenberger at
Fri Jan 12 15:44:27 CET 2007

Can you attach files in this forum? Couldn't find the option. Oh well,
here's the file.

# Version: 1.1
# Author:  Steve Losh

from sets import Set
from optparse import OptionParser
from xml.dom.minidom import parse

AudioPath     = 'audio/'
DatafilePath  = 'utterances.trmxml'
CONFIDENCE_LOW  = None #'500'

utterancesFile = None

class Utterance:
    def __init__(self, audio, grammarSet, text):      = audio
        self.grammarSet = grammarSet
        self.text       = text

    def __str__(self):
        return "SWIrecAcousticStateReset\ntranscription " + self.text \
                + "\nrecognize " + AudioPath +

def getGrammarPaths():
    """Get the paths of all the grammars needed.  Returns a Set
containing the results.
If a grammar is listed more than once in the transcription manifest it
will only appear once in these results.

Find a less fragile way to split off the server half of the URIs."""
    grammarTags  = utterancesFile.getElementsByTagName('Grammar')
    grammarURIs  = [tag.getAttribute('uri') for tag in grammarTags]
    grammarPaths = [uri.split('servlet/CA/')[1] for uri in grammarURIs]
    return Set(grammarPaths)

def createGrammarNameFromPath(path):
    """Convert a given path into an appropriate name for the
    path = path.replace('/', '-')  # Strip the directory slashes
    path = path.replace('.', '_')  # and the dot before the extension.
    return path

def loadGrammars():
    """Output the statements that will load the required grammars."""
    grammarPaths = list(getGrammarPaths())
    grammarsToLoad = {}
    for path in grammarPaths:
        grammarName = createGrammarNameFromPath(path)
        grammarsToLoad[grammarName] = "grammars/" + path
    for grammarName in grammarsToLoad:
        print "SWIrecGrammarLoad", grammarName,

def loadGrammarSets():
    """Output the statements that will define the grammar
sets/contexts.  Returns a list of the grammar set names."""
    grammarSetList  =
    grammarSets     =
    grammarSetNames = []
    for gs in grammarSets:
        grammarSetName = gs.getAttribute('id')
        print "context_define", grammarSetName, CONFIDENCE_LOW,
        for g in gs.getElementsByTagName('Grammar'):
            path = g.getAttribute('uri').split('servlet/CA/')[1]
            print "context_add", createGrammarNameFromPath(path),
        print "context_end\n"
    return grammarSetNames

def buildUtterances(call):
    """This function takes a call tag, builds the utterances belonging
to it and returns a list containing them."""
    utts = call.getElementsByTagName('Utt')
    utterances = [Utterance( utt.getAttribute('audio'), \
                             utt.getAttribute('transcribedText') ) \
                  for utt in utts]
    return utterances

def getUtterances():
    """Returns a list of all the utterances we want to test."""
    callList = utterancesFile.getElementsByTagName('Calls')[0]
    calls = callList.getElementsByTagName('Call')
    utterances = []
    for c in calls:
    return utterances

def loadData(utterances):
    """Outputs the statements that will tell rec_test what to test."""
    contexts = {}
    for u in utterances:
        if u.grammarSet not in contexts:
            contexts[u.grammarSet] = []
    for c in contexts:
        print "open errors " + c + ".errors"
        print "open utd "    + c + ".utd"
        print "context_use", c
        for u in contexts[c]:
	    print u
        print "close utd"
        print "close errors"
        print "\n"

def makeParser():
    parser = OptionParser( "usage: %prog -l LOWER CONFIDENCE -h UPPER
CONFIDENCE [-f FILTER1 -f FILTER2 ...] file" )
    parser.add_option("-l", "--low-confidence", dest="lower", \
                      help="The lower confidence level to test at.",
    parser.add_option("-u", "--upper-confidence", dest="upper", \
                      help="The upper confidence level to test at.",
    parser.add_option("-f", "--filter", dest="filter", action="append",
                      help="Only test utterances transcribed as WORD.",
    return parser

def main():
    global utterancesFile, CONFIDENCE_HIGH, CONFIDENCE_LOW
    parser = makeParser()
    (options, args) = parser.parse_args()
    if len(args) != 1:
        parser.error("One data file must be specified.")
    elif options.lower == None:
        parser.error("A lower confidence level must be specified.")
    elif options.upper == None:
        parser.error("An upper confidence level must be specified.")

    DatafilePath = args[0]
    CONFIDENCE_LOW  = options.lower
    CONFIDENCE_HIGH = options.upper

    utterancesFile = parse(DatafilePath)

    print ':ACC\n\n'
    print "\n\n"

    grammarSetNames = loadGrammarSets()

    utterances = getUtterances()
    if options.filter != None:
        utterances = [u for u in utterances if u.text in

    print "\n\n"

    print "report summary summary.txt"
    print "report oov oov.txt"
    print "report words words.txt"

if __name__ == '__main__':

Peter Otten wrote:
> gheissenberger at wrote:
> > HELP!
> > Guy who was here before me wrote a script to parse files in Python.
> >
> > Includes line:
> > print u
> According to your other posts 'u' seems to be an instance of a custom
> Utterance class with a __str__() method that accidentally returns unicode.
> Try changing the print statement to
> print unicode(u)
> If you're lucky, it works. Otherwise we need a piece of the actual code. To
> give you an idea what a self-contained demonstration of your problem might
> look like:
> >>> class Utterance(object):
> ...     def __str__(self): return u"äöü"
> ...
> >>> u = Utterance()
> >>> print u
> Traceback (most recent call last):
>   File "<stdin>", line 1, in <module>
> UnicodeEncodeError: 'ascii' codec can't encode characters in position 0-2:
> ordinal not in range(128)
> >>> print unicode(u)
> äöü
> Peter

More information about the Python-list mailing list