[Twisted-Python] Memory Usage Problem

Sept. 6, 2010

      Hi All,

I have a problem which is that memory usage of twisted is increasing but not 
decreasing over time.

After few days, it reaches 520MB. You may find the codes below;
How may I reduce the usage of memory?
Twisted version: 8.2.0
Python version: 2.4.3

[root@labris init.d]# cat /usr/bin/labris-flyng.tac
#!/usr/bin/python

import os

from twisted.application import internet, service
from twisted.internet import reactor
from twisted.enterprise import adbapi
from twisted.plugin import getPlugins
from twisted.python import log

import labris.flyng.iflyng as iflyng
import labris.flyng.config as config
import labris.flyng.plugins as pplugins
import labris.flyng.protocols as flyng_protocols

from labris.flyng.config import flyng_config

# Database type to package dictionary
DB_PACKAGE_DICT = {
    "postgresql" : "psycopg2"
}

def initDatabase(config):
    """
    Create database connection pool
    """

    dbtype = config.getDBType()
    dbpackage = DB_PACKAGE_DICT.get(dbtype, None)

    if not dbpackage:
        print "Database type % is not supported yet!" %(dbtype)
        sys.exit(1)

    if dbtype == "mysql":
        dbpool = adbapi.ConnectionPool(
            dbpackage,
                        config.getDBHost(),
                        config.getDBUser(),
                        config.getDBPassword(),
                        config.getDBName()
                )
    else:
            dbpool = adbapi.ConnectionPool(
            dbpackage,
                    database=config.getDBName(),
                    user=config.getDBUser(),
            password=config.getDBPassword(),
                    host=config.getDBHost()
            )

    dbpool.connect()

    return dbpool

def setupPlugins(config, dbpool):
    """
    Initialize and setup plugins
    """

    for plugin in getPlugins(iflyng.IFlyngPlugin, pplugins):
        print "* Plugin %s" %(plugin.name)
        if not config.isEnabled(plugin.name):
            print "   - Not enabled"
            continue

        plugin.init(config, dbpool)

        for type, port in config.getListeningPorts(plugin.name):
            port = port.replace('"', "")
            if type == "tcp":
                print "   -> TCP port: %s" %(port)
                factory = flyng_protocols.FlyngServerFactory(plugin)
                internet.TCPServer(
                    int(port),
                    factory
                ).setServiceParent(application)

            elif type == "udp":
                print "   -> UDP port: %s" %(port)
                internet.UDPServer(
                    int(port),
                    flyng_protocols.FlyngUDPProtocol(plugin)
                ).setServiceParent(application)

            elif type == "unix":
                print "   -> UNIX stream: %s" %(port)
                if os.path.exists(port):
                    os.unlink(port)
                factory = flyng_protocols.FlyngServerFactory(plugin)
                internet.UNIXServer(
                    port,
                    factory
                ).setServiceParent(application)

            else:
                raise "connection type %s is not supported" %(port)

#log.startLogging(open("/var/log/labris-flyng.log", "a"))

config = config.Configuration()

uid, gid = config.getRunAs()

application = service.Application('labris-flyng', uid, gid)

dbpool = initDatabase(config)

setupPlugins(config, dbpool)

-------------------------------------------------------------------

[root@labris init.d]# cat /usr/bin/labris-pyng.tac
#!/usr/bin/python

"""
    A syslog-ng to database mapping service
"""

import os

from twisted.internet import protocol, reactor, defer
from twisted.application import internet, service
from twisted.python import log

from labris.logutils import factories
from labris.logutils.config import pyng_config

class LogFactory(protocol.ServerFactory):
    pass

class UnixLogFactory(protocol.ServerFactory):
    pass

def openUNIXStreams(configuration, application):
    for logtype, stream in configuration.getUnixStreams():
        try:
            if not pyng_config.getboolean(logtype.strip(), "enabled"):
                print "Warning: UNIX stream: ", logtype, " is DISABLED"
                continue

            print "\nPlugin: %s [UNIX-Stream(%s)]" %(logtype, stream)

            factory = UnixLogFactory()
            factory.protocol = 
factories.ProtocolFactory.getLogProtocol(logtype.strip(), "stream")

            # delete old stream
            if os.path.exists(stream):
                os.unlink(stream)

            internet.UNIXServer(stream.strip(), 
factory).setServiceParent(application)
        except factories.NoSuchProtocolException, e:
            print "Error: no such protocol:", e.getProtoname()

    for logtype, sock in configuration.getUnixDgrams():
        try:
            if not pyng_config.getboolean(logtype.strip(), "enabled"):
                print "Warning: UNIX dgram: ", logtype, " is DISABLED"
                continue

            print "\nPlugin: %s [UNIX-Datagram(%s)]" %(logtype, stream)

            protocol = factories.ProtocolFactory.getLogProtocol(logtype.strip(), 
"datagram")

            # delete old stream
            if os.path.exists(sock):
                os.unlink(sock)

            internet.UNIXDatagramServer(sock.strip(), 
protocol()).setServiceParent(application)
        except factories.NoSuchProtocolException, e:
            print "Error: no such protocol:", e.getProtoname()

def openSockets(configuration, application):
    for logtype, hostname, port in configuration.getTCP():
        try:
            if not pyng_config.getboolean(logtype.strip(), "enabled"):
                print "Warning: TCP socket: ", logtype, " is DISABLED"
                continue

            print "\nPlugin: %s [TCP => port:%d]" %(logtype, int(port))

            factory = LogFactory()
            factory.protocol = 
factories.ProtocolFactory.getLogProtocol(logtype.strip(), "stream")

            internet.TCPServer(int(port), factory).setServiceParent(application)
        except factories.NoSuchProtocolException, e:
            print "Error: no such protocol:", e.getProtoname()

    for logtype, hostname, port in configuration.getUDP():
        try:
            if not pyng_config.getboolean(logtype.strip(), "enabled"):
                print "Warning: UDP socket: ", logtype, " is DISABLED"
                continue

            print "\nPlugin: %s [UDP => port:%d]" %(logtype, int(port))

            protocol = factories.ProtocolFactory.getLogProtocol(logtype.strip(), 
"datagram")

            internet.UDPServer(int(port), 
protocol()).setServiceParent(application)
        except factories.NoSuchProtocolException, e:
            print "Error: no such protocol:", e.getProtoname()

#log.startLogging(open("/var/log/labris-pyng.log", "a"))

from labris.logutils.dao.pyngdb import PyNgDB

PyNgDB.initDatabase(pyng_config.bind())

application = service.Application('labris-pyng', pyng_config.uid(), 
pyng_config.gid())

openUNIXStreams(pyng_config, application)
openSockets(pyng_config, application)

[Twisted-Python] Memory Usage Problem

Ozgur Pekcagliyan