Differences between revisions 1 and 2
Revision 1 as of 2009-05-19 16:31:19
Size: 4958
Editor: PeterFein
Comment:
Revision 2 as of 2009-05-19 16:46:06
Size: 5010
Editor: PeterFein
Comment:
Deletions are marked like this. Additions are marked like this.
Line 5: Line 5:
Take one down, pass it around,
Line 6: Line 7:
Take one down, pass it around, 98 bottles of beer on the wall, 98 bottles of beer

99 Concurrency Bottles of Beer

99 bottles of beer on the wall, 99 bottles of beer.
Take one down, pass it around,
Take one down, pass it around,
97 bottles of beer on the wall, 97 bottles of beer
98 bottles of beer on the wall, 98 bottles of beer

Solutions to common concurrent problems in different styles/toolkits. Inspired by 99 Bottles of Beer.

Include your name and a brief description if you add to this page. Please make sure your source is well commented - concurrency is hard!

The problem

Implement

#!/bin/sh
tail -f /var/log/system.log |grep pants

in concurrent Python. On unix, you can send syslog messages via logger; filenames may vary.

Generator

Generators implement a "pull-style" approach to concurrency.

   1 import time
   2 import re
   3 
   4 def follow(fname):
   5     f = file(fname)
   6     f.seek(0,2) # go to the end
   7     while True:
   8         l = f.readline()
   9         if not l: # no data
  10             time.sleep(.1)
  11         else:
  12             yield l
  13 
  14 def grep(lines, pattern):
  15     regex = re.compile(pattern)
  16     for l in lines:
  17         if regex.match(l):
  18             yield l
  19 
  20 def printer(lines):
  21     for l in lines:
  22         print l.strip()
  23 
  24 f = follow('/var/log/system.log')
  25 g = grep(f, ".*pants.*")
  26 p = printer(g)
  27 
  28 for i in p:
  29     pass

Coroutines

The inversion of the generator example above, coroutines use a "push-style" approach to concurrency:

   1 import time
   2 import re
   3 from functools import wraps
   4 
   5 
   6 def coroutine(func):
   7     @wraps(func)
   8     def thing(*args, **kwargs):
   9         gen = func(*args, **kwargs)
  10         gen.next() # advance to the first yield
  11         return gen
  12     return thing
  13 
  14 @coroutine
  15 def follow(fname, next):
  16     f = file(fname)
  17     f.seek(0,2) # go to the end
  18     while True:
  19         l = f.readline()
  20         if not l: # no data
  21             time.sleep(.1)
  22         else:
  23             next.send(l)
  24 
  25 @coroutine
  26 def grep(pattern, next):
  27     regex = re.compile(pattern)
  28     while True:
  29         l = yield
  30         if regex.match(l):
  31             next.send(l)
  32 
  33 @coroutine
  34 def printer():
  35     while True:
  36         l = yield
  37         print l.strip()
  38         
  39 
  40 p = printer()
  41 g = grep('.*pants.*', p)
  42 f = follow('/var/log/system.log', g)

Greenlets

Greenlets are similar to coroutines.

   1 import greenlet
   2 import time
   3 import re
   4 
   5 def follow(fname, next):
   6     # setup
   7     f = file(fname)
   8     f.seek(0,2) # go to the end
   9 
  10     # do stuff
  11     while True:
  12         l = f.readline()
  13         if not l: # no data
  14             time.sleep(.1)
  15         else:
  16             next.switch(l)
  17     
  18 def grep(pattern, next):
  19     # setup
  20     regex = re.compile(pattern)
  21     
  22     def do_stuff(l):
  23         parent = greenlet.getcurrent().parent
  24         
  25         while True:
  26             if regex.match(l):
  27                 l = next.switch(l)
  28             else:
  29                 l = parent.switch() # subtle!
  30 
  31     return do_stuff
  32 
  33 def printer(l):
  34     # no setup
  35     parent = greenlet.getcurrent().parent
  36 
  37     # do stuff
  38     while True:
  39         print l.strip()
  40         l = parent.switch()
  41 
  42 # parents == main greenlet - HERE!
  43 g_printer = greenlet.greenlet(printer) 
  44 g_grep = greenlet.greenlet(grep(".*pants.*", g_printer))
  45 
  46 follow("/var/log/system.log", g_grep)

Kamelia

   1 import time
   2 import re
   3 
   4 import Axon
   5 from Kamaelia.Chassis.Pipeline import Pipeline
   6 
   7 # threaded due to the time.sleep() call
   8 # No yield since a threaded component
   9 class Follow(Axon.ThreadedComponent.threadedcomponent):
  10     def __init__(self, fname, **argv):
  11         self.fname = fname
  12         super(Follow,self).__init__(**argv)
  13     def main(self):
  14         f = file(self.fname)
  15         f.seek(0,2) # go to the end
  16         while not self.dataReady("control"):
  17             l = f.readline()
  18             if not l: # no data
  19                 time.sleep(.1)
  20             else:
  21                 self.send(l, "outbox")
  22 
  23         self.send(self.recv("control"), "signal")
  24 
  25 class Grep(Axon.Component.component):
  26     # Default pattern, override in constructor with pattern="some pattern"
  27     # See below
  28     pattern = "." 
  29     def main(self):
  30         regex = re.compile(self.pattern)
  31         while not self.dataReady("control"):
  32            for l in self.Inbox("inbox"):
  33                if regex.match(l):
  34                    self.send(l, "outbox")
  35            self.pause()
  36            yield 1
  37         self.send(self.recv("control"), "signal")
  38 
  39 class Printer(Axon.Component.component):
  40     def main(self):
  41         while not self.dataReady("control"):
  42             for l in self.Inbox("inbox"):
  43                 print l.strip()
  44             self.pause()
  45             yield 1
  46         self.send(self.recv("control"), "signal")
  47 
  48 Pipeline(
  49     Follow('tail -f /var/log/system.log'),
  50     Grep(".*pants.*"),
  51     Printer(),
  52 ).run()

Concurrency/99Bottles (last edited 2014-10-14 21:57:17 by ppaez)

Unable to edit the page? See the FrontPage for instructions.