let mapReduce partition distribute process combine input =
partition input |> distribute process |> combine
let par_mapReduce ?process_count ~combine ~process =
let process_count = process_count |? !global_process_count in
mapReduce (splitInto process_count) (par_map ~process_count) process combine
let pfoldlSeqN ?process_count n r f init =
mapReduce (groupsOf n) (flip foldl init)
(fun acc l -> r acc (pfoldl ?process_count r f init l)) id
let piterSeqN ?process_count n r f =
mapReduce (groupsOf n) iter (iter r @. pmap ?process_count f) ignore
Compared to the old versions of p*SeqN it's quite WTF indeed. Does it buy us anything? Heck do I know. A headache, maybe?
(* Old versions *)
let pfoldlSeqN ?process_count n r f init l =
foldl (fun acc il -> r acc (pfoldl ?process_count r f init il)) init (groupsOf n l)
let piterSeqN ?process_count n r f l =
iter (fun l -> iter r (pmap ?process_count f l)) (groupsOf n l)
[edit] Reverted back to the two above definitions. They're shorter and cleaner.
No comments:
Post a Comment