multiprocessing, what am I doing wrong?
Eric Frederich
eric.frederich at gmail.com
Mon Feb 27 11:57:20 EST 2012
Still freezing sometimes, like 1 out of 10 times that I run it.
Here is updated code and a couple of outputs.
################ code
#!/usr/bin/env python
import sys
import Queue
import multiprocessing
import time
def FOO(a, b, c):
print 'foo', a, b, c
return (a + b) * c
class MyWorker(multiprocessing.Process):
def __init__(self, name, inbox, outbox):
super(MyWorker, self).__init__()
self.name = name
self.inbox = inbox
self.outbox = outbox
print >> sys.stderr, 'Created %s' % self.name; sys.stderr.flush()
def run(self):
print >> sys.stderr, 'Running %s' % self.name; sys.stderr.flush()
while True:
try:
args = self.inbox.get_nowait()
print >> sys.stderr, '%s got something to do' % self.name;
sys.stderr.flush()
except Queue.Empty:
break
self.outbox.put(FOO(*args))
if __name__ == '__main__':
# This file is being run as the main script. This part won't be
# run if the file is imported.
print >> sys.stderr, 'Creating todo queue'; sys.stderr.flush()
todo = multiprocessing.Queue()
for i in xrange(100):
todo.put((i, i+1, i+2))
print >> sys.stderr, 'Creating results queue'; sys.stderr.flush()
result_queue = multiprocessing.Queue()
print >> sys.stderr, 'Creating Workers'; sys.stderr.flush()
w1 = MyWorker('Worker 1', todo, result_queue)
w2 = MyWorker('Worker 2', todo, result_queue)
print >> sys.stderr, 'Starting Worker 1'; sys.stderr.flush()
w1.start()
print >> sys.stderr, 'Starting Worker 2'; sys.stderr.flush()
w2.start()
for i in xrange(100):
print result_queue.get()
################ output 1 (I ctrl-c'd it after it froze)
Creating todo queue
Creating results queue
Creating Workers
Created Worker 1
Created Worker 2
Starting Worker 1
Running Worker 1
Starting Worker 2
Running Worker 2
Traceback (most recent call last):
File "./multi.py", line 53, in <module>
print result_queue.get()
File
"/home/frede00e/software/python/lib/python2.7/multiprocessing/queues.py",
line 91, in get
res = self._recv()
KeyboardInterrupt
################### output 2
Creating todo queue
Creating results queue
Creating Workers
Created Worker 1
Created Worker 2
Starting Worker 1
Running Worker 1
Worker 1 got something to do
Starting Worker 2
foo 0 1 2
Worker 1 got something to do
foo 1 2 3
Worker 1 got something to do
foo 2 3 4
Worker 1 got something to do
foo 3 4 5
Worker 1 got something to do
foo 4 5 6
Worker 1 got something to do
foo 5 6 7
Worker 1 got something to do
foo 6 7 8
Worker 1 got something to do
foo 7 8 9
Worker 1 got something to do
foo 8 9 10
Worker 1 got something to do
foo 9 10 11
Worker 1 got something to do
foo 10 11 12
Worker 1 got something to do
foo 11 12 13
Worker 1 got something to do
foo 12 13 14
Worker 1 got something to do
foo 13 14 15
Worker 1 got something to do
foo 14 15 16
Worker 1 got something to do
foo 15 16 17
Worker 1 got something to do
foo 16 17 18
Worker 1 got something to do
foo 17 18 19
Running Worker 2
2
9
20
35
54
77
104
135
170
209
252
299
350
405
464
527
594
665
Traceback (most recent call last):
File "./multi.py", line 53, in <module>
print result_queue.get()
File
"/home/frede00e/software/python/lib/python2.7/multiprocessing/queues.py",
line 91, in get
res = self._recv()
KeyboardInterrupt
On Fri, Feb 24, 2012 at 1:36 PM, MRAB <python at mrabarnett.plus.com> wrote:
> On 24/02/2012 17:00, Eric Frederich wrote:
>
>> I can sill get it to freeze and nothing is printed out from the other
>> except block.
>> Does it look like I'm doing anything wrong here?
>>
>> [snip]
> I don't normally use multiprocessing, so I forgot about a critical
> detail. :-(
>
> When the multiprocessing module starts a process, that process
> _imports_ the module which contains the function which is to be run, so
> what's happening is that when your script is run, it creates and starts
> workers, the multiprocessing module makes a new process for each
> worker, each of those processes then imports the script, which creates
> and starts workers, etc, leading to an ever-increasing number of
> processes.
>
> The solution is to ensure that the script/module distinguishes between
> being run as the main script and being imported as a module:
>
>
> #!/usr/bin/env python
>
> import sys
> import Queue
> import multiprocessing
> import time
>
> def FOO(a, b, c):
> print 'foo', a, b, c
> return (a + b) * c
>
> class MyWorker(multiprocessing.**Process):
> def __init__(self, inbox, outbox):
> super(MyWorker, self).__init__()
> self.inbox = inbox
> self.outbox = outbox
> print >> sys.stderr, '1' * 80; sys.stderr.flush()
> def run(self):
> print >> sys.stderr, '2' * 80; sys.stderr.flush()
> while True:
> try:
> args = self.inbox.get_nowait()
> except Queue.Empty:
> break
> self.outbox.put(FOO(*args))
>
> if __name__ == '__main__':
> # This file is being run as the main script. This part won't be
> # run if the file is imported.
>
> todo = multiprocessing.Queue()
>
> for i in xrange(100):
> todo.put((i, i+1, i+2))
>
> print >> sys.stderr, 'a' * 80; sys.stderr.flush()
> result_queue = multiprocessing.Queue()
>
> print >> sys.stderr, 'b' * 80; sys.stderr.flush()
> w1 = MyWorker(todo, result_queue)
> print >> sys.stderr, 'c' * 80; sys.stderr.flush()
> w2 = MyWorker(todo, result_queue)
>
> print >> sys.stderr, 'd' * 80; sys.stderr.flush()
> w1.start()
> print >> sys.stderr, 'e' * 80; sys.stderr.flush()
> w2.start()
> print >> sys.stderr, 'f' * 80; sys.stderr.flush()
>
> for i in xrange(100):
> print result_queue.get()
> --
> http://mail.python.org/**mailman/listinfo/python-list<http://mail.python.org/mailman/listinfo/python-list>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.python.org/pipermail/python-list/attachments/20120227/470c604a/attachment-0001.html>
More information about the Python-list
mailing list