Dataflow: Declarative concurrency in Ruby

Dataflow
The declarative concurrent
programming model

Larry Diehl

{:larrytheliquid =>
%w[.com
github
twitter]}

Outline

Purpose of presentation

Gradual explanation of concepts

Helpful tips

Lexical Scope

foo = :foo
define_method :foo do
foo
end

Dynamic Scope

def foo
@foo
end

Mutability

def initialize
@foo = :foo
end

def foo
@foo
end

Mutability

def foo
@foo = :foo
@foo
end

Mutability+Concurrency

def initialize
Thread.new { loop { @foo = :shazbot } }
end

def foo
@foo = :foo
@foo
end

Declarative Synchronous

my_var = :bound
my_var = :rebind # NOT ALLOWED!


local do |my_var|
my_var.object_id # thread sleeps
end


local do |my_var|
unify my_var, :bound
unify my_var, :rebind # =>
# Dataflow::UnificationError,
# ":bound != :rebind"
end


class MyClass
declare :my_var
def initialize
unify my_var, :bound
end
end

Declarative Concurrent
(MAGIC)

Declarative Concurrent

local do |my_var|
Thread.new { unify my_var, :bound }
my_var.should == :bound
end

Dependency Resolution

local do |sentence, middle, tail|
Thread.new { unify middle, "base are belong #{tail}" }
Thread.new { unify tail, "to us" }
Thread.new { unify sentence, "all your #{middle}" }
sentence.should == "all your base are belong to us"
end

Asynchronous Output
def Worker.async(output=nil)
Thread.new do
result = # do hard work
unify output, result if output
end
end

local do |output|
Worker.async(output)
output.should == # hard work result
end

Asynchronous Output
local do |output|
flow(output) do
# do hard work
end
output.should == # hard work result
end

Anonymous variables

{'google.com' => Dataflow::Variable.new,
'bing.com' => Dataflow::Variable.new
}.map do |domain,var|
Thread.new do
unify var, open("http://#{domain}").read
end
var
end

need_later

%w[google.com bing.com].map do |domain|
need_later { open("http://#{domain}").read }
end

Chunked Sequential Processing

(1..100).each_slice(10).map do |chunk|
sleep(1)
chunk.inject(&:+)
end.inject(&:+) # => ~10s

Chunked Parallel Processing

(1..100).each_slice(10).map do |chunk|
need_later do
sleep(1)
chunk.inject(&:+)
end
end.inject(&:+) # => ~1s

Leaving Declarative
via Async

Ports & Streams

local do |port, stream|
unify port, Dataflow::Port.new(stream)
port.send 1
port.send 2
stream.take(2).should == [1, 2]
end

Ports & Streams (async)
local do |port, stream|
unify port, Dataflow::Port.new(stream)
Thread.new do
stream.each do |message|
puts "received: #{message}"
end
end
%w[x y z].each do |letter|
Thread.new{ port.send letter }
end
stream.take(3).sort.should == %w[x y z]
end

FutureQueue
local do |queue, first, second, third|
unify queue, FutureQueue.new
queue.pop first
queue.pop second
queue.push 1
queue.push 2
queue.push 3
queue.pop third
[first, second, third].should == [1, 2, 3]
end

Actors
Ping = Actor.new { Pong = Actor.new {
3.times { 3.times {
case receive case receive
when :ping when :pong
puts "Ping" puts "Pong"
Pong.send :pong Ping.send :ping
end end
} }
} }

Ping.send :ping

by_need

def baz(num)
might_get_used = by_need { Factory.gen }
might_get_used.value if num%2 == 0
end

Modular

local do |my_var|
Thread.new { unify my_var, :bound }
# my_var.wait
end

Debugging

local do |my_var|
my_var.inspect
# => #<Dataflow::Variable:2637860 unbound>
end

Class/Module methods
Dataflow.local do |my_var|
Dataflow.async do
Dataflow.unify my_var, :bound
end
end

Use Cases
general purpose
concurrency for elegant program structure with respect
to coordination

concurrency to make use of extra processors/cores
(depending on Ruby implementation)

web development
worker daemons

concurrently munging together data from various rest
api's

Ruby Implementations

Pure Ruby library, should work on any implementation

JRuby in particular has a great GC, no GIL, native threads,
and a tunable threadpool option.

Rubinius has more code written in Ruby, so it proxies more
method calls (e.g. Array#flatten).

class FutureQueue
include Dataflow
declare :push_port, :pop_port

def initialize
local do |pushed, popped|
unify push_port, Dataflow::Port.new(pushed)
unify pop_port, Dataflow::Port.new(popped)

Thread.new {
loop do
barrier pushed.head, popped.head
unify popped.head, pushed.head
pushed, popped = pushed.tail, popped.tail
end
}
end
end

def push(x) push_port.send x end
def pop(x) pop_port.send x end
end

The End
sudo port install dataflow

http://github.com
/larrytheliquid
/dataflow

freenode: #dataflow-gem

Dataflow: Declarative concurrency in Ruby

Recommended

More Related Content

What's hot (20)

Similar to Dataflow: Declarative concurrency in Ruby (20)

Recently uploaded (20)

Dataflow: Declarative concurrency in Ruby