[Numpy-discussion] RAM problem during code execution - Numpya arrays
Francesc Alted
francesc at continuum.io
Fri Aug 23 10:34:06 EDT 2013
Hi José,
The code is somewhat longish for a pure visual inspection, but my advice is
that you install memory profiler (
https://pypi.python.org/pypi/memory_profiler). This will help you
determine which line or lines are hugging the memory the most.
Saludos,
Francesc
On Fri, Aug 23, 2013 at 3:58 PM, Josè Luis Mietta <
joseluismietta at yahoo.com.ar> wrote:
> Hi ecperts. I need your help with a RAM porblem during execution of my
> script.
> I wrote the next code. I use SAGE. In 1-2 hours of execution time the RAM
> of my laptop (8gb) is filled and the sistem crash:
>
> from scipy.stats import uniform import numpy as np
>
> cant_de_cadenas =[700,800,900]
>
> cantidad_de_cadenas=np.array([])
> for kkkkk in cant_de_cadenas:
> cantidad_de_cadenas=np.append(cantidad_de_cadenas,kkkkk)
>
> cantidad_de_cadenas=np.transpose(cantidad_de_cadenas)
>
> b=10
> h=bLongitud=1
> numero_experimentos=150
>
> densidad_de_cadenas =cantidad_de_cadenas/(b**2)
>
> prob_perc=np.array([])
>
> tiempos=np.array([])
>
> S_int=np.array([])
>
> S_medio=np.array([])
>
> desviacion_standard=np.array([])
>
> desviacion_standard_nuevo=np.array([])
>
> anisotropia_macroscopica_porcentual=np.array([])
>
> componente_y=np.array([])
>
> componente_x=np.array([])
> import time
> for N in cant_de_cadenas:
>
> empieza=time.clock()
>
> PERCOLACION=np.array([])
>
> size_medio_intuitivo = np.array([])
> size_medio_nuevo = np.array([])
> std_dev_size_medio_intuitivo = np.array([])
> std_dev_size_medio_nuevo = np.array([])
> comp_y = np.array([])
> comp_x = np.array([])
>
>
> for u in xrange(numero_experimentos):
>
>
> perco = False
>
> array_x1=uniform.rvs(loc=-b/2, scale=b, size=N)
> array_y1=uniform.rvs(loc=-h/2, scale=h, size=N)
> array_angle=uniform.rvs(loc=-0.5*(np.pi), scale=np.pi, size=N)
>
>
> array_pendiente_x=1./np.tan(array_angle)
>
> random=uniform.rvs(loc=-1, scale=2, size=N)
> lambda_sign=np.zeros([N])
> for t in xrange(N):
> if random[t]<0:
> lambda_sign[t]=-1
> else:
> lambda_sign[t]=1
> array_lambdas=(lambda_sign*Longitud)/np.sqrt(1+array_pendiente_x**2)
>
>
> array_x2= array_x1 + array_lambdas*array_pendiente_x
> array_y2= array_y1 + array_lambdas*1
>
> array_x1 = np.append(array_x1, [-b/2, b/2, -b/2, -b/2])
> array_y1 = np.append(array_y1, [-h/2, -h/2, -h/2, h/2])
> array_x2 = np.append(array_x2, [-b/2, b/2, b/2, b/2])
> array_y2 = np.append(array_y2, [h/2, h/2, -h/2, h/2])
>
> M = np.zeros([N+4,N+4])
>
> for j in xrange(N+4):
> if j>0:
> x_A1B1 = array_x2[j]-array_x1[j]
> y_A1B1 = array_y2[j]-array_y1[j]
> x_A1A2 = array_x1[0:j]-array_x1[j]
> y_A1A2 = array_y1[0:j]-array_y1[j]
> x_A2A1 = -1*x_A1A2
> y_A2A1 = -1*y_A1A2
> x_A2B2 = array_x2[0:j]-array_x1[0:j]
> y_A2B2 = array_y2[0:j]-array_y1[0:j]
> x_A1B2 = array_x2[0:j]-array_x1[j]
> y_A1B2 = array_y2[0:j]-array_y1[j]
> x_A2B1 = array_x2[j]-array_x1[0:j]
> y_A2B1 = array_y2[j]-array_y1[0:j]
>
> p1 = x_A1B1*y_A1A2 - y_A1B1*x_A1A2
> p2 = x_A1B1*y_A1B2 - y_A1B1*x_A1B2
> p3 = x_A2B2*y_A2B1 - y_A2B2*x_A2B1
> p4 = x_A2B2*y_A2A1 - y_A2B2*x_A2A1
>
> condicion_1=p1*p2
> condicion_2=p3*p4
>
> for k in xrange (j):
> if condicion_1[k]<=0 and condicion_2[k]<=0:
> M[j,k]=1
> del condicion_1
> del condicion_2
>
>
> if j+1<N+4:
> x_A1B1 = array_x2[j]-array_x1[j]
> y_A1B1 = array_y2[j]-array_y1[j]
> x_A1A2 = array_x1[j+1:]-array_x1[j]
> y_A1A2 = array_y1[j+1:]-array_y1[j]
> x_A2A1 = -1*x_A1A2
> y_A2A1 = -1*y_A1A2
> x_A2B2 = array_x2[j+1:]-array_x1[j+1:]
> y_A2B2 = array_y2[j+1:]-array_y1[j+1:]
> x_A1B2 = array_x2[j+1:]-array_x1[j]
> y_A1B2 = array_y2[j+1:]-array_y1[j]
> x_A2B1 = array_x2[j]-array_x1[j+1:]
> y_A2B1 = array_y2[j]-array_y1[j+1:]
>
> p1 = x_A1B1*y_A1A2 - y_A1B1*x_A1A2
> p2 = x_A1B1*y_A1B2 - y_A1B1*x_A1B2
> p3 = x_A2B2*y_A2B1 - y_A2B2*x_A2B1
> p4 = x_A2B2*y_A2A1 - y_A2B2*x_A2A1
>
> condicion_1=p1*p2
> condicion_2=p3*p4
>
> for k in xrange ((N+4)-j-1):
> if condicion_1[k]<=0 and condicion_2[k]<=0:
> M[j,k+j+1]=1
> del condicion_1
> del condicion_2
>
> M[N,N+2]=0
> M[N,N+3]=0
> M[N+1,N+2]=0
> M[N+1,N+3]=0
> M[N+2,N]=0
> M[N+2,N+1]=0
> M[N+3,N]=0
> M[N+3,N+1]=0
>
>
> CD=np.array([])
>
> POPOPO=[]
> for g in xrange(N):
>
> lala=0
> r=False
> while lala<=len(POPOPO)-1:
> esta= g in POPOPO[lala]
>
> if esta is True:
> lala=len(POPOPO)
> r=True
> else:
> lala=lala+1
> if r is False:
>
>
>
> L=np.array([g])
> for s in xrange(N):
> if M[g,s] != 0:
> L=np.append(L,s)
>
> x=0
> while x<= N:
> for l in xrange(N):
> z= l in L
> d=L[x]
> if z is False and M[d,l] != 0:
> L=np.append(L,l)
> if x+1<len(L):
> x+=1
> else:
> x=N+1.
>
> q= len (L)
> CD=np.append(CD, q)
> POPOPO.append(L)
>
>
> M_horizontal=M.copy()
> M_horizontal[:,N+2] = np.zeros(N+4)
> M_horizontal[:,N+3] = np.zeros(N+4)
> M_horizontal[N+2] = np.zeros(N+4)
> M_horizontal[N+3] = np.zeros(N+4)
>
>
>
> L=np.array([N])
> for s in xrange(N+4):
> if M_horizontal[N,s] != 0:
> L=np.append(L,s)
>
> x=0
> while x<= N+4:
> for l in xrange(N+4):
> z= l in L
> d=L[x]
> if z is False and M_horizontal[d,l] != 0:
> L=np.append(L,l)
> if x+1<len(L):
> x+=1
> else:
> x=(N+4)+1.
>
> LV1_in_L = N in L
> LV2_in_L= (N+1) in L
>
> if LV1_in_L is True and LV2_in_L is True:
> perc_horiz=True
> else:
> perc_horiz=False
>
>
> M_vertical=M.copy()
>
> M_vertical[:,N] = np.zeros(N+4)
> M_vertical[:,N+1] = np.zeros(N+4)
> M_vertical[N] = np.zeros(N+4)
> M_vertical[N+1] = np.zeros(N+4)
>
>
>
> L=np.array([N+2])
> for s in xrange(N+4):
> if M_vertical[N+2,s] != 0:
> L=np.append(L,s)
>
> x=0
> while x<= N+4:
> for l in xrange(N+4):
> z= l in L
> d=L[x]
> if z is False and M_vertical[d,l] != 0:
> L=np.append(L,l)
> if x+1<len(L):
> x+=1
> else:
> x=(N+4)+1.
>
> LH1_in_L = (N+2) in L
> LH2_in_L= (N+3) in L
>
> if LH1_in_L is True and LH2_in_L is True:
> perc_ver = True
> else:
> perc_ver = False
>
>
>
> if perc_ver is True or perc_horiz is True:
> PERCOLACION=np.append(PERCOLACION,1)
> perco=True
>
>
> D = np.array([])
> W = np.array([])
>
> for c in xrange (int(min(CD)), int(max(CD)+1),1):
> D=np.append(D,c)
> frec = sum (CD == c)
> W = np.append(W,frec)
>
> if perco is True:
> posicion=np.argmax(D)
> D=np.delete(D,posicion)
> W=np.delete(W,posicion)
>
> if len(D) == 0 and len(W)==0:
> S_medio_intuitivo_exp_u=0
> S_medio_nuevo_exp_u = 0
> std_dev_exp_u = 0
> std_dev_nuevo_exp_u = 0
> else:
>
> S_medio_intuitivo_exp_u = np.average (D,weights=W)
>
> peso_nuevo=D*W
>
> S_medio_nuevo_exp_u = np.average (D,weights=peso_nuevo)
>
>
> tipos=sum(W)
> X=W*((D-S_medio_intuitivo_exp_u)**2)
> S=sum(X)
>
> std_dev_exp_u = np.sqrt(S/(tipos-1.))
>
> tipos_nuevo=sum(peso_nuevo)
> X_nuevo=peso_nuevo*((D-S_medio_nuevo_exp_u)**2)
> S_nuevo=sum(X_nuevo)
>
> std_dev_nuevo_exp_u = np.sqrt(S_nuevo/(tipos_nuevo-1.))
>
>
> componente_longitudinal=Longitud*np.abs(np.cos(array_angle))
> comp_y=np.append(comp_y, sum(componente_longitudinal)/N)
>
> componente_transversal=Longitud*np.abs(np.sin(array_angle))
> comp_x=np.append(comp_x, sum(componente_transversal)/N)
>
> std_dev_size_medio_intuitivo=np.append(std_dev_size_medio_intuitivo, std_dev_exp_u)
>
> std_dev_size_medio_nuevo=np.append(std_dev_size_medio_nuevo, std_dev_nuevo_exp_u)
>
> size_medio_intuitivo=np.append(size_medio_intuitivo, S_medio_intuitivo_exp_u)
>
> size_medio_nuevo=np.append(size_medio_nuevo, S_medio_nuevo_exp_u)
>
>
> percolation_probability=sum(PERCOLACION)/numero_experimentos
>
> prob_perc=np.append(prob_perc, percolation_probability)
>
> S_int = np.append (S_int, sum(size_medio_intuitivo)/numero_experimentos)
>
> S_medio=np.append (S_medio, sum(size_medio_nuevo)/numero_experimentos)
>
> desviacion_standard = np.append (desviacion_standard, sum(std_dev_size_medio_intuitivo)/numero_experimentos)
>
> desviacion_standard_nuevo=np.append (desviacion_standard_nuevo, sum(std_dev_size_medio_nuevo)/numero_experimentos)
>
> tiempos=np.append(tiempos, time.clock()-empieza)
>
> componente_y=np.append(componente_y, sum(comp_y)/numero_experimentos)
> componente_x=np.append(componente_x, sum(comp_x)/numero_experimentos)
>
> anisotropia_macroscopica_porcentual=100*(1-(componente_y/componente_x))
>
> I tryed with gc and gc.collect() and 'del'command for deleting arrays
> after his use and nothing work!
>
> What am I doing wrong? Why the memory becomes full while running (starts
> with 10% of RAM used and in 1-2hour is totally full used)?
>
> Please help me, I'm totally stuck!
> Thanks a lot!
>
> _______________________________________________
> NumPy-Discussion mailing list
> NumPy-Discussion at scipy.org
> http://mail.scipy.org/mailman/listinfo/numpy-discussion
>
>
--
Francesc Alted
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.python.org/pipermail/numpy-discussion/attachments/20130823/63c94c96/attachment.html>
More information about the NumPy-Discussion
mailing list