memory error

[questions anon]

Hello All,
I keep coming across a memory error when processing many netcdf files. I
assume it has something to do with how I loop things and maybe need to close
things off properly.
In the code below I am looping through a bunch of netcdf files (each file is
hourly data for one month) and within each netcdf file I am outputting a
*png file every three hours.
This works for one netcdf file but when it begins to process the next netcdf
file I receive this memory error:

*Traceback (most recent call last):
  File
"d:/plot_netcdf_merc_multiplot_across_multifolders_mkdirs_memoryerror.py",
line 44, in <module>
    TSFC=ncfile.variables['T_SFC'][:]
  File "netCDF4.pyx", line 2473, in netCDF4.Variable.__getitem__
(netCDF4.c:23094)
MemoryError*

To reduce processing requirements I have tried making the LAT and LON to
only use [0] but I also receive an error:

*Traceback (most recent call last):
  File
"d:/plot_netcdf_merc_multiplot_across_multifolders_mkdirs_memoryerror.py",
line 75, in <module>
    x,y=map(*N.meshgrid(LON,LAT))
  File "C:\Python27\lib\site-packages\numpy\lib\function_base.py", line
3256, in meshgrid
    numRows, numCols = len(y), len(x)  # yes, reversed
TypeError: len() of unsized object*

finally I have added gc.collect() in a couple of places but that doesn't
seem to do anything to help.
I am using :*Python 2.7.2 |EPD 7.1-2 (32-bit)| (default, Jul  3 2011,
15:13:59) [MSC v.1500 32 bit (Intel)] on win32*
Any feedback will be greatly appreciated!


from netCDF4 import Dataset
import numpy
import numpy as N
import matplotlib.pyplot as plt
from numpy import ma as MA
from mpl_toolkits.basemap import Basemap
from netcdftime import utime
from datetime import datetime
import os
import gc

print "start processing...."

inputpath=r'E:/GriddedData/Input/'
outputpath=r'E:/GriddedData/Validation/'
shapefile1="E:/test_GIS/DSE_REGIONS"
for (path, dirs, files) in os.walk(inputpath):
    for dir in dirs:
        print dir
        sourcepath=os.path.join(path,dir)
        relativepath=os.path.relpath(sourcepath,inputpath)
        newdir=os.path.join(outputpath,relativepath)
        if not os.path.exists(newdir):
            os.makedirs(newdir)

    for ncfile in files:
        if ncfile[-3:]=='.nc':
            print "dealing with ncfiles:", ncfile
            ncfile=os.path.join(sourcepath,ncfile)
            #print ncfile
            ncfile=Dataset(ncfile, 'r+', 'NETCDF4')
            TSFC=ncfile.variables['T_SFC'][:,:,:]
            TIME=ncfile.variables['time'][:]
            LAT=ncfile.variables['latitude'][:]
            LON=ncfile.variables['longitude'][:]
            fillvalue=ncfile.variables['T_SFC']._FillValue
            TSFC=MA.masked_values(TSFC, fillvalue)
            ncfile.close()
            gc.collect()
            print "garbage collected"


            for TSFC, TIME in zip((TSFC[1::3]),(TIME[1::3])):
                print TSFC, TIME
            #convert time from numbers to date and prepare it to have no
symbols for saving to filename
                cdftime=utime('seconds since 1970-01-01 00:00:00')
                ncfiletime=cdftime.num2date(TIME)
                print ncfiletime
                timestr=str(ncfiletime)
                d = datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S')
                date_string = d.strftime('%Y%m%d_%H%M')

                #Set up basemap using mercator projection
http://matplotlib.sourceforge.net/basemap/doc/html/users/merc.html
                map = Basemap(projection='merc',llcrnrlat=-40,urcrnrlat=-33,

llcrnrlon=139.0,urcrnrlon=151.0,lat_ts=0,resolution='i')

            # compute map projection coordinates for lat/lon grid.
                x,y=map(*N.meshgrid(LON,LAT))
                map.drawcoastlines(linewidth=0.5)
                map.readshapefile(shapefile1, 'DSE_REGIONS')
                map.drawstates()

                plt.title('Surface temperature at %s UTC'%ncfiletime)
                ticks=[-5,0,5,10,15,20,25,30,35,40,45,50]
                CS = map.contourf(x,y,TSFC, ticks, cmap=plt.cm.jet)
                l,b,w,h =0.1,0.1,0.8,0.8
                cax = plt.axes([l+w+0.025, b, 0.025, h], )
                cbar=plt.colorbar(CS, cax=cax, drawedges=True)

            #save map as *.png and plot netcdf file

plt.savefig((os.path.join(newdir,'TSFC'+date_string+'UTC.png')))
                plt.close()
                gc.collect()
                print "garbage collected again"
print "end of processing"
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.python.org/pipermail/python-list/attachments/20110914/3f2fec93/attachment.html>