art with code

2008-09-12

prelude.ml: further modularization

Created List, Array, String and ByteString -modules, pasted the parallelization shunt in. And fixed some stupid bugs (yay for inline unit tests.) Thinking of putting all the new functions in the modules, moving the modules to top and writing a long list of aliases for convenience in Prelude.Pervasives.

The parallelization shunt is a piece of code that I've been copypasting in the collection modules (List, Array, etc.) It gets you the parallel combinators as long as you have a compatible module signature (I'm using copypaste as a ghetto functor.) The functions you need for all the combinators are: splitInto, foldl, foldl1, foldr, foldr1, iter, map, filter, concat, groupsOf, init, length, and, for pzipWith, either unsafe_get or zipWith, depending on whether you have an array-like or a list-like data structure:

let par_mapReduce ?process_count ~combine ~process l =
let process_count = process_count |? !global_process_count in
splitInto process_count l |> par_map ~process_count process |> combine

let pmapReduce combine process = par_mapReduce ~combine ~process

let pfoldl r f init = pmapReduce (List.foldl1 r) (foldl f init)
let pfoldl1 f = pmapReduce (List.foldl1 f) (foldl1 f)
let pfoldr r f init = pmapReduce (List.foldr1 r) (foldr f init)
let pfoldr1 f = pmapReduce (List.foldr1 f) (foldr1 f)

let piter f = pmapReduce ignore (iter f)
let pmap f = pmapReduce concat (map f)
let pfilter f = pmapReduce concat (filter f)

let pfoldlSeqN ?process_count n r f init l =
List.foldl (fun acc il -> r acc (pfoldl ?process_count r f init il))
init (groupsOf n l)

let piterSeqN ?process_count n r f l =
List.iter (fun l -> iter r (pmap ?process_count f l)) (groupsOf n l)

let pinit ?process_count f l =
let process_count = process_count |? !global_process_count in
let plen = int (ceil (float l /. float process_count)) in
let process i =
let start = plen * i in
let len = min plen (l - start) in
init (fun j -> f (start + j)) len in
concat (par_map ~process_count process (0--(process_count-1)))

(* for array-likes *)
let pzipWith ?process_count f a b =
let process_count = process_count |? !global_process_count in
let len = min (length a) (length b) in
pinit ~process_count (fun i ->
f (unsafe_get a i) (unsafe_get b i)
) len

(* for list-likes *)
let pzipWith ?process_count f a b =
let process_count = process_count |? !global_process_count in
let len = min (len a) (len b) in
let plen = int (ceil (float len /. float process_count)) in
let aspl = groupsOf plen a in
let bspl = groupsOf plen b in
concat (par_map ~process_count (uncurry (zipWith f)) (zip aspl bspl))

No comments:

Blog Archive