[Numpy-discussion] RAM problem during code execution - Numpya arrays
Benjamin Root
ben.root at ou.edu
Fri Aug 23 10:59:10 EDT 2013
On Fri, Aug 23, 2013 at 10:34 AM, Francesc Alted <francesc at continuum.io>wrote:
> Hi José,
>
> The code is somewhat longish for a pure visual inspection, but my advice
> is that you install memory profiler (
> https://pypi.python.org/pypi/memory_profiler). This will help you
> determine which line or lines are hugging the memory the most.
>
> Saludos,
> Francesc
>
> On Fri, Aug 23, 2013 at 3:58 PM, Josè Luis Mietta <
> joseluismietta at yahoo.com.ar> wrote:
>
>> Hi ecperts. I need your help with a RAM porblem during execution of my
>> script.
>> I wrote the next code. I use SAGE. In 1-2 hours of execution time the RAM
>> of my laptop (8gb) is filled and the sistem crash:
>>
>> from scipy.stats import uniform import numpy as np
>>
>> cant_de_cadenas =[700,800,900]
>>
>> cantidad_de_cadenas=np.array([])
>> for kkkkk in cant_de_cadenas:
>> cantidad_de_cadenas=np.append(cantidad_de_cadenas,kkkkk)
>>
>> cantidad_de_cadenas=np.transpose(cantidad_de_cadenas)
>>
>> b=10
>> h=bLongitud=1
>> numero_experimentos=150
>>
>> densidad_de_cadenas =cantidad_de_cadenas/(b**2)
>>
>> prob_perc=np.array([])
>>
>> tiempos=np.array([])
>>
>> S_int=np.array([])
>>
>> S_medio=np.array([])
>>
>> desviacion_standard=np.array([])
>>
>> desviacion_standard_nuevo=np.array([])
>>
>> anisotropia_macroscopica_porcentual=np.array([])
>>
>> componente_y=np.array([])
>>
>> componente_x=np.array([])
>> import time
>> for N in cant_de_cadenas:
>>
>> empieza=time.clock()
>>
>> PERCOLACION=np.array([])
>>
>> size_medio_intuitivo = np.array([])
>> size_medio_nuevo = np.array([])
>> std_dev_size_medio_intuitivo = np.array([])
>> std_dev_size_medio_nuevo = np.array([])
>> comp_y = np.array([])
>> comp_x = np.array([])
>>
>>
>> for u in xrange(numero_experimentos):
>>
>>
>> perco = False
>>
>> array_x1=uniform.rvs(loc=-b/2, scale=b, size=N)
>> array_y1=uniform.rvs(loc=-h/2, scale=h, size=N)
>> array_angle=uniform.rvs(loc=-0.5*(np.pi), scale=np.pi, size=N)
>>
>>
>> array_pendiente_x=1./np.tan(array_angle)
>>
>> random=uniform.rvs(loc=-1, scale=2, size=N)
>> lambda_sign=np.zeros([N])
>> for t in xrange(N):
>> if random[t]<0:
>> lambda_sign[t]=-1
>> else:
>> lambda_sign[t]=1
>> array_lambdas=(lambda_sign*Longitud)/np.sqrt(1+array_pendiente_x**2)
>>
>>
>> array_x2= array_x1 + array_lambdas*array_pendiente_x
>> array_y2= array_y1 + array_lambdas*1
>>
>> array_x1 = np.append(array_x1, [-b/2, b/2, -b/2, -b/2])
>> array_y1 = np.append(array_y1, [-h/2, -h/2, -h/2, h/2])
>> array_x2 = np.append(array_x2, [-b/2, b/2, b/2, b/2])
>> array_y2 = np.append(array_y2, [h/2, h/2, -h/2, h/2])
>>
>> M = np.zeros([N+4,N+4])
>>
>> for j in xrange(N+4):
>> if j>0:
>> x_A1B1 = array_x2[j]-array_x1[j]
>> y_A1B1 = array_y2[j]-array_y1[j]
>> x_A1A2 = array_x1[0:j]-array_x1[j]
>> y_A1A2 = array_y1[0:j]-array_y1[j]
>> x_A2A1 = -1*x_A1A2
>> y_A2A1 = -1*y_A1A2
>> x_A2B2 = array_x2[0:j]-array_x1[0:j]
>> y_A2B2 = array_y2[0:j]-array_y1[0:j]
>> x_A1B2 = array_x2[0:j]-array_x1[j]
>> y_A1B2 = array_y2[0:j]-array_y1[j]
>> x_A2B1 = array_x2[j]-array_x1[0:j]
>> y_A2B1 = array_y2[j]-array_y1[0:j]
>>
>> p1 = x_A1B1*y_A1A2 - y_A1B1*x_A1A2
>> p2 = x_A1B1*y_A1B2 - y_A1B1*x_A1B2
>> p3 = x_A2B2*y_A2B1 - y_A2B2*x_A2B1
>> p4 = x_A2B2*y_A2A1 - y_A2B2*x_A2A1
>>
>> condicion_1=p1*p2
>> condicion_2=p3*p4
>>
>> for k in xrange (j):
>> if condicion_1[k]<=0 and condicion_2[k]<=0:
>> M[j,k]=1
>> del condicion_1
>> del condicion_2
>>
>>
>> if j+1<N+4:
>> x_A1B1 = array_x2[j]-array_x1[j]
>> y_A1B1 = array_y2[j]-array_y1[j]
>> x_A1A2 = array_x1[j+1:]-array_x1[j]
>> y_A1A2 = array_y1[j+1:]-array_y1[j]
>> x_A2A1 = -1*x_A1A2
>> y_A2A1 = -1*y_A1A2
>> x_A2B2 = array_x2[j+1:]-array_x1[j+1:]
>> y_A2B2 = array_y2[j+1:]-array_y1[j+1:]
>> x_A1B2 = array_x2[j+1:]-array_x1[j]
>> y_A1B2 = array_y2[j+1:]-array_y1[j]
>> x_A2B1 = array_x2[j]-array_x1[j+1:]
>> y_A2B1 = array_y2[j]-array_y1[j+1:]
>>
>> p1 = x_A1B1*y_A1A2 - y_A1B1*x_A1A2
>> p2 = x_A1B1*y_A1B2 - y_A1B1*x_A1B2
>> p3 = x_A2B2*y_A2B1 - y_A2B2*x_A2B1
>> p4 = x_A2B2*y_A2A1 - y_A2B2*x_A2A1
>>
>> condicion_1=p1*p2
>> condicion_2=p3*p4
>>
>> for k in xrange ((N+4)-j-1):
>> if condicion_1[k]<=0 and condicion_2[k]<=0:
>> M[j,k+j+1]=1
>> del condicion_1
>> del condicion_2
>>
>> M[N,N+2]=0
>> M[N,N+3]=0
>> M[N+1,N+2]=0
>> M[N+1,N+3]=0
>> M[N+2,N]=0
>> M[N+2,N+1]=0
>> M[N+3,N]=0
>> M[N+3,N+1]=0
>>
>>
>> CD=np.array([])
>>
>> POPOPO=[]
>> for g in xrange(N):
>>
>> lala=0
>> r=False
>> while lala<=len(POPOPO)-1:
>> esta= g in POPOPO[lala]
>>
>> if esta is True:
>> lala=len(POPOPO)
>> r=True
>> else:
>> lala=lala+1
>> if r is False:
>>
>>
>>
>> L=np.array([g])
>> for s in xrange(N):
>> if M[g,s] != 0:
>> L=np.append(L,s)
>>
>> x=0
>> while x<= N:
>> for l in xrange(N):
>> z= l in L
>> d=L[x]
>> if z is False and M[d,l] != 0:
>> L=np.append(L,l)
>> if x+1<len(L):
>> x+=1
>> else:
>> x=N+1.
>>
>> q= len (L)
>> CD=np.append(CD, q)
>> POPOPO.append(L)
>>
>>
>> M_horizontal=M.copy()
>> M_horizontal[:,N+2] = np.zeros(N+4)
>> M_horizontal[:,N+3] = np.zeros(N+4)
>> M_horizontal[N+2] = np.zeros(N+4)
>> M_horizontal[N+3] = np.zeros(N+4)
>>
>>
>>
>> L=np.array([N])
>> for s in xrange(N+4):
>> if M_horizontal[N,s] != 0:
>> L=np.append(L,s)
>>
>> x=0
>> while x<= N+4:
>> for l in xrange(N+4):
>> z= l in L
>> d=L[x]
>> if z is False and M_horizontal[d,l] != 0:
>> L=np.append(L,l)
>> if x+1<len(L):
>> x+=1
>> else:
>> x=(N+4)+1.
>>
>> LV1_in_L = N in L
>> LV2_in_L= (N+1) in L
>>
>> if LV1_in_L is True and LV2_in_L is True:
>> perc_horiz=True
>> else:
>> perc_horiz=False
>>
>>
>> M_vertical=M.copy()
>>
>> M_vertical[:,N] = np.zeros(N+4)
>> M_vertical[:,N+1] = np.zeros(N+4)
>> M_vertical[N] = np.zeros(N+4)
>> M_vertical[N+1] = np.zeros(N+4)
>>
>>
>>
>> L=np.array([N+2])
>> for s in xrange(N+4):
>> if M_vertical[N+2,s] != 0:
>> L=np.append(L,s)
>>
>> x=0
>> while x<= N+4:
>> for l in xrange(N+4):
>> z= l in L
>> d=L[x]
>> if z is False and M_vertical[d,l] != 0:
>> L=np.append(L,l)
>> if x+1<len(L):
>> x+=1
>> else:
>> x=(N+4)+1.
>>
>> LH1_in_L = (N+2) in L
>> LH2_in_L= (N+3) in L
>>
>> if LH1_in_L is True and LH2_in_L is True:
>> perc_ver = True
>> else:
>> perc_ver = False
>>
>>
>>
>> if perc_ver is True or perc_horiz is True:
>> PERCOLACION=np.append(PERCOLACION,1)
>> perco=True
>>
>>
>> D = np.array([])
>> W = np.array([])
>>
>> for c in xrange (int(min(CD)), int(max(CD)+1),1):
>> D=np.append(D,c)
>> frec = sum (CD == c)
>> W = np.append(W,frec)
>>
>> if perco is True:
>> posicion=np.argmax(D)
>> D=np.delete(D,posicion)
>> W=np.delete(W,posicion)
>>
>> if len(D) == 0 and len(W)==0:
>> S_medio_intuitivo_exp_u=0
>> S_medio_nuevo_exp_u = 0
>> std_dev_exp_u = 0
>> std_dev_nuevo_exp_u = 0
>> else:
>>
>> S_medio_intuitivo_exp_u = np.average (D,weights=W)
>>
>> peso_nuevo=D*W
>>
>> S_medio_nuevo_exp_u = np.average (D,weights=peso_nuevo)
>>
>>
>> tipos=sum(W)
>> X=W*((D-S_medio_intuitivo_exp_u)**2)
>> S=sum(X)
>>
>> std_dev_exp_u = np.sqrt(S/(tipos-1.))
>>
>> tipos_nuevo=sum(peso_nuevo)
>> X_nuevo=peso_nuevo*((D-S_medio_nuevo_exp_u)**2)
>> S_nuevo=sum(X_nuevo)
>>
>> std_dev_nuevo_exp_u = np.sqrt(S_nuevo/(tipos_nuevo-1.))
>>
>>
>> componente_longitudinal=Longitud*np.abs(np.cos(array_angle))
>> comp_y=np.append(comp_y, sum(componente_longitudinal)/N)
>>
>> componente_transversal=Longitud*np.abs(np.sin(array_angle))
>> comp_x=np.append(comp_x, sum(componente_transversal)/N)
>>
>> std_dev_size_medio_intuitivo=np.append(std_dev_size_medio_intuitivo, std_dev_exp_u)
>>
>> std_dev_size_medio_nuevo=np.append(std_dev_size_medio_nuevo, std_dev_nuevo_exp_u)
>>
>> size_medio_intuitivo=np.append(size_medio_intuitivo, S_medio_intuitivo_exp_u)
>>
>> size_medio_nuevo=np.append(size_medio_nuevo, S_medio_nuevo_exp_u)
>>
>>
>> percolation_probability=sum(PERCOLACION)/numero_experimentos
>>
>> prob_perc=np.append(prob_perc, percolation_probability)
>>
>> S_int = np.append (S_int, sum(size_medio_intuitivo)/numero_experimentos)
>>
>> S_medio=np.append (S_medio, sum(size_medio_nuevo)/numero_experimentos)
>>
>> desviacion_standard = np.append (desviacion_standard, sum(std_dev_size_medio_intuitivo)/numero_experimentos)
>>
>> desviacion_standard_nuevo=np.append (desviacion_standard_nuevo, sum(std_dev_size_medio_nuevo)/numero_experimentos)
>>
>> tiempos=np.append(tiempos, time.clock()-empieza)
>>
>> componente_y=np.append(componente_y, sum(comp_y)/numero_experimentos)
>> componente_x=np.append(componente_x, sum(comp_x)/numero_experimentos)
>>
>> anisotropia_macroscopica_porcentual=100*(1-(componente_y/componente_x))
>>
>> I tryed with gc and gc.collect() and 'del'command for deleting arrays
>> after his use and nothing work!
>>
>> What am I doing wrong? Why the memory becomes full while running (starts
>> with 10% of RAM used and in 1-2hour is totally full used)?
>>
>> Please help me, I'm totally stuck!
>> Thanks a lot!
>>
>>
With the following code in the very beginning (not the cause):
cant_de_cadenas =[700,800,900]
cantidad_de_cadenas=np.array([])
for kkkkk in cant_de_cadenas:
cantidad_de_cadenas=np.append(cantidad_de_cadenas,kkkkk)
cantidad_de_cadenas=np.transpose(cantidad_de_cadenas)
That can'e be a very good sign. The thing to remember about np.append() is
that it only exists as a convenience and should be used as infrequently as
possible. Calling append makes a new copy each time. Instead, if you know
the length of the array ahead of time, initialize those arrays to those
sizes before looping and just assign the value into them. For example,
with the above example from your code, I would write that as
"cantidad_de_cadenas = np.array([[700, 800, 900]])". A lot of the code you
have here can be greatly simplified. I would start with just trying to get
rid of appends as much as possible and use preallocated arrays with
np.empty() or np.ones() or the likes.
Cheers!
Ben Root
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.python.org/pipermail/numpy-discussion/attachments/20130823/232ced56/attachment.html>
More information about the NumPy-Discussion
mailing list