which library has map reduce and how to use it for this case

Ho Yeung Lee davidbenny2000 at gmail.com
Thu Jun 9 21:07:04 EDT 2016


input are these six operators, output is finding full column of 27 elements add together is 54

i got memory error when searching this, 

i have difficulty in understanding map reduce and transforming my program into map reduce problem

M1 = {}
M2 = {}
M3 = {}
M4 = {}
M5 = {}
V6 = {}
M1['00']=0
M1['01']=2
M1['02']=1
M1['10']=1
M1['11']=1
M1['12']=1
M1['20']=1
M1['21']=1
M1['22']=2
M2['00']=0
M2['01']=1
M2['02']=1
M2['10']=1
M2['11']=1
M2['12']=1
M2['20']=1
M2['21']=1
M2['22']=1
M3['00']=2
M3['01']=2
M3['02']=2
M3['10']=0
M3['11']=2
M3['12']=1
M3['20']=0
M3['21']=1
M3['22']=2
M4['00']=1
M4['01']=2
M4['02']=1
M4['10']=2
M4['11']=2
M4['12']=2
M4['20']=0
M4['21']=1
M4['22']=2
M5['00']=0
M5['01']=1
M5['02']=1
M5['10']=0
M5['11']=2
M5['12']=1
M5['20']=0
M5['21']=1
M5['22']=1
V6['00']=1
V6['01']=1
V6['02']=2
V6['10']=1
V6['11']=2
V6['12']=1
V6['20']=1
V6['21']=2
V6['22']=2
MM = {}
MM[0] = M1
MM[1] = M2
MM[2] = M3
MM[3] = M4
MM[4] = M5
MM[5] = V6
m = 3
b = [str(i)+str(j)+str(k) for i in range(m) for j in range(m) for k in range(m)]
import itertools
deep = 3
final = []
mylist = [MM[i] for i in range(0,7-1)]
b = [str(i)+str(j)+str(k) for i in range(m) for j in range(m) for k in range(m)]
def DFS(b, deep, maxx, sourceoperators, path):
 initlist = []
 if deep > 0:
  print("deep=", deep)
  for aa,bb in itertools.combinations(sourceoperators, 2):
   print(aa,bb)
   if deep == maxx:
    finalresult = []
    op1xy = [aa[b[i][0:1]+b[i][1:2]] for i in range(len(b))]
    op1yz = [aa[b[i][1:2]+b[i][2:3]] for i in range(len(b))]
    op1xz = [aa[b[i][0:1]+b[i][2:3]] for i in range(len(b))]
    op2xy = [bb[b[i][0:1]+b[i][1:2]] for i in range(len(b))]
    op2yz = [bb[b[i][1:2]+b[i][2:3]] for i in range(len(b))]
    op2xz = [bb[b[i][0:1]+b[i][2:3]] for i in range(len(b))]
    if sum(op1xy) == 54:
      path.append([(deep, aa, "xy")])
    if sum(op1yz) == 54:
      path.append([(deep, aa, "yz")])
    if sum(op1xz) == 54:
      path.append([(deep, aa, "xz")])
    if sum(op2xy) == 54:
      path.append([(deep, bb, "xy")])
    if sum(op2yz) == 54:
      path.append([(deep, bb, "yz")])
    if sum(op2xz) == 54:      
      path.append([(deep, bb, "xz")])
    initlist.append(op1xy)
    initlist.append(op1yz)
    initlist.append(op1xz)
    initlist.append(op2xy)
    initlist.append(op2yz)
    initlist.append(op2xz)
   else:
    level = []
    for j in range(len(b)):
     op1xy = [aa[b[j][i]] for i in range(len(b[j]))]
     op2xy = [bb[b[j][i]] for i in range(len(b[j]))]
     if sum(op1xy) == 54:
      path.append([(deep, aa, "xy")])
     if sum(op2xy) == 54:
      path.append([(deep, bb, "xy")])
     level.append(op1xy)
     level.append(op2xy)
     initlist.append(op1xy)
     initlist.append(op2xy)
 if deep == maxx:
  b = []
  #print(len(list(itertools.combinations(initlist, 2))))
  for aaa,bbb in itertools.combinations(initlist, 2):     
   b.append([str(i)+str(j) for i,j in zip(aaa, bbb)])
  path = DFS(b, deep-1, maxx, sourceoperators, path)
 else:
  #print(len(list(itertools.combinations(initlist, 2))))
  for aaa,bbb in itertools.combinations(initlist, 2):
   b.append([str(i)+str(j) for i,j in zip(aaa, bbb)])
  path = DFS(b, deep-1, maxx, sourceoperators, path)
 return path

path = []
mresult = DFS(b, 2, 2, mylist, path)




Michael Selik於 2016年6月10日星期五 UTC+8上午6時45分14秒寫道:
> I like using Yelp's mrjob module (https://github.com/Yelp/mrjob) to run
> Python on Hadoop.
> 
> On Thu, Jun 9, 2016 at 2:56 AM Ho Yeung Lee <davidbenny2000 at gmail.com>
> wrote:
> 
> > [... a bunch of code ...]
> 
> 
> If you want to describe a map-reduce problem, start with the data. What
> does a record of your input data look like?
> 
> Then think about your mapper. What key-value pairs will you extract from
> each line of data?
> 
> Then think about your reducer. For a single key and its associated values,
> what will you calculate?



More information about the Python-list mailing list