reading large file

[guillaume]

I have to read and process a large ASCII file containing a mesh : a
list of points and triangles.
The file is 100 MBytes.
 
I first tried to do it in memory but I think I am running out of
memory therefore I decide to use the shelve
module to store my points and elements on disks.
Despite the fact it is slow ... Any hint ? I think I have the same
memory problem but I don't understand why
since  my aPoint should be removed by the gc.

Have you any idea ?

Thanks

Guillaume

PS :
here is the code for your info




import string
import os
import sys
import time
import resource
import shelve
import psyco
 
psyco.full()
 
class point:
  def __init__(self,x,y,z):
    self.x = x
    self.y = y
    self.z = z
    
 
def SFMImport(filename):
  print 'UNV Import ("%s")' % filename
 
  db = shelve.open('points.db')
 
  file = open(filename, "r")
 
  linenumber = 1
  nbpoints   = 0
  nbfaces    = 0
 
  pointList = []
  faceList  = []
 
  line  = file.readline()
  words = string.split(line)
  nbpoints = string.atoi(words[1])
  nbtrias  = string.atoi(words[0])
 
  print "found %s points and %s triangles" % (nbpoints, nbtrias)
 
  t1 = time.time()
  for i in range(nbpoints):
    line  = file.readline()
    words = string.split(line)
 
    x = string.atof(words[1].replace("D","E"))
    y = string.atof(words[2].replace("D","E"))
    z = string.atof(words[3].replace("D","E"))
 
    aPoint = point(x, y, z)
 
    as = "point%s" % i
 
    if (i%250000 == 0):
      print "%7d points <%s>" % (i, time.time() - t1)
      t1 = time.time()
 
    db[as] = aPoint
 
  print "%s points read in %s seconds" % (nbpoints, time.time() - t1)
  bd.close()
 
  t1 = time.time()
  t2 = time.time()
  for i in range(nbtrias):
    line  = file.readline()
    words = string.split(line)
 
    i1    = string.atoi(words[0])
    i2    = string.atoi(words[1])
    i3    = string.atoi(words[2])
 
    faceList.append((i1,i2,i3))
 
    if (i%100000 == 0):
      print "%s faces <%s>" % (i, time.time() - t1)
      t1 = time.time()
 
  print "%s points read in %s seconds" % (nbpoints, time.time() - t2)
 
  file.close()
 
def callback(fs):
    filename = fs.filename
    UNVImport(filename)

              
if __name__ == "__main__":
#    try:
#        import GUI
#    except:
#        print "This script is only working with the new GUI module
..."
#    else:
#        fs = GUI.FileSelector()
#        fs.activate(callback, fs)
  print sys.argv[0]
  SFMImport(sys.argv[1])