Memory problems (garbage collection)

Carbon Man darwin at
Thu Apr 23 08:50:23 CEST 2009

Very new to Python, running 2.5 on windows.
I am processing an XML file (7.2MB). Using the standard library I am 
recursively processing each node and parsing it. The branches don't go 
particularly deep. What is happening is that the program is running really 
really slowly, so slow that even running it over night, it still doesn't 
Stepping through it I have noticed that memory usage has shot up from 190MB 
to 624MB and continues to climb. If I set a break point and then stop the 
program the memory is not released. It is not until I shutdown PythonWin 
that the memory gets released.
I thought this might mean objects were not getting GCed, so through the 
interactive window I imported gc. gc.garbage is empty. gc.collect() seems to 
fix the problem (after much thinking) and reports 2524104. Running it again 
returns 0.
I thought that garbage collection was automatic, if I use variables in a 
method do I have to del them?
I tried putting a "del node" in all my for node in .... loops but that 
didn't help. collect() reports the same number. Tried putting gc.collect() 
at the end of the loops but that didn't help either.
If I have the program at a break and do gc.collect() it doesn't fix it, so 
whatever referencing is causing problems is still active.
My program is parsing the XML and generating a Python program for 
SQLalchemy, but the program never gets a chance to run the memory problem is 
prior to that. It probably has something to do with the way I am string 

My apologies for the long post but without being able to see the code I 
doubt anyone can give me a solid answer so here it goes (sorry for the lack 
of comments):

from xml.dom import minidom
import os
import gc

class xmlProcessing:
    """ General class for XML processing"""

    def process(self, filename="", xmlString=""):
        if xmlString:
        elif filename:
            xmldoc = minidom.parse(filename)
        self.parse( xmldoc.documentElement )

    def parseBranch(self, parentNode):
        """ Process an XML branch """
        for node in parentNode.childNodes:
                parseMethod = getattr(self, "parse_%s" % 
            except AttributeError:
            if parseMethod(node):
            del node

    def parse_Document(self, node):

    def parse_Text(self, node):

    def parse_Comment(self, node):

    def parse_Element(self, node):
            handlerMethod = getattr(self, "do_%s" % node.tagName)
        except AttributeError:
            return False
        return True

class reptorParsing(xmlProcessing):
    """ Specific class for generating a SQLalchemy program to create tables
    and populate them with data"""

    def __init__(self):
        self.schemaPreface = """from sqlalchemy import *
from sqlalchemy.ext.declarative import declarative_base
engine = create_engine('sqlite:///tutorial.db', echo=False)
metadata = MetaData()
Base = declarative_base()"""
        self.schemaTables = ""
        self.schemaFields = ""
        self.dataUpdate = ""
        self.tableDict = {}
        self.tableName = ""
        self.tables = ""

    def parse(self, parentNode):
        """Main entry point to begin processing a XML document"""
        # Properties such as schemaTables and .tables are populated by the 
various methods below
        fupdate=open(os.path.join(os.getcwd(), ""), 'w')
        if self.schemaTables:
            fupdate.write("import schema\n")
            f=open(os.path.join(os.getcwd(), ""), 'w')
                '\n' + "metadata.create_all(engine)\n"+
                "print 'hello 2'")
        if self.tables:
#            f=open(os.path.join(os.getcwd(), ""), 'w')
#            f.write(self.dataUpdate)
#            f.close()

    def do_TABLES(self, tableNode):
        """Process schema for tables"""
        for node in tableNode.childNodes:
            self.tableName = node.tagName
            # Define a declaritive mapping class
            self.schemaTables += """\nclass %s(Base):
    __tablename__ = '%s'
""" % (self.tableName, self.tableName)
            self.schemaFields = ""
            # allow for userA = users("Billy","Bob") via a __init__()
            self.schemaInitPreface = "    def __init__(self"
            self.schemaInitBody = ""
            self.schemaInitPreface += "):\n"
            self.schemaTables += self.schemaFields + "\n" + \
                self.schemaInitPreface + \
                self.schemaInitBody + "\n"

    def do_FIELDS(self, fieldsNode):
        """Process schema for fields within tables"""
        for node in fieldsNode.childNodes:
            if self.schemaFields:
                self.schemaFields += "\n"
            cType = ""
            # The attribute type holds the type of field
            crType = node.attributes["type"].value
            if crType==u"C":
                cType = "String(length=%s)" % node.attributes["len"].value
            elif crType==u"N" and node.attributes["dec"].value==u'0':
                cType = "Integer"
            elif crType==u"N":
                cType = "Numeric(precision=%s, scale=%s)" % 
            elif crType==u"L":
                cType = "Boolean"
            elif crType==u"T":
                cType = "DateTime"
            elif crType==u"D":
                cType = "Date"
            elif crType==u"M" or crType==u"G":
                cType = "Text"

            if node.attributes.getNamedItem("primary"):
                cType += ", primary_key=True"
            self.schemaFields += "    %s = Column(%s)" % (node.tagName, 
            self.schemaInitPreface += ", \\\n        %s" % (node.tagName)
            self.schemaInitBody += "            self.%s = %s\n" % 
(node.tagName, node.tagName)
            self.tableDict[self.tableName + "." + node.tagName] = crType
            del node

    def do_DATA(self, dataNode):
        """This is for processing actual data to be pushed into the tables

        Layout is DATA -> TABLE_NAME key='primary_field' -> TUPLE -> 
        for node in dataNode.childNodes:
            self.dataUpdate = """
import time
from datetime import *
from sqlalchemy import *
from sqlalchemy.orm import *
engine = create_engine('sqlite:///tutorial.db', echo=False)
Session = sessionmaker()
session = Session()
            self.keyValue = ""
            self.keyField = node.attributes["key"].value
            self.tableName = node.tagName
            self.tables += "\nimport" % (self.tableName)
            f=open(os.path.join(os.getcwd(), self.tableName + ""), 

    def do_TUPLE(self, tupleNode):
        """ A TUPLE is what the XML file refers to a table row
        Sits below a DATA child"""
        self.dataUpdate += """
entry = %s()
""" % (self.tableName)
        for node in tupleNode.childNodes:
            for dataNode in node.childNodes:
                crType = self.tableDict[self.tableName + "." + node.tagName]

                if crType==u"C" or crType==u"M":
                    cValue = '"""%s"""' %
                elif crType==u"T":
                    cValue = 'datetime.strptime("''", 
"%Y-%m-%d %H:%M")'
                elif crType==u"D":
                    cValue = 'datetime.strptime("''", 
                    cValue =

                self.dataUpdate += "\nentry.%s = %s" % (node.tagName, 
                del dataNode

        self.dataUpdate += "\nsession.commit()"
        del node

if __name__ == '__main__':
    replicate = reptorParsing()
    replicate.process(filename=os.path.join(os.getcwd(), "request.xml"))
    import update 

More information about the Python-list mailing list