art with code

2009-01-04

Random thought on automated testing

An automatic test generator that takes the function signature would be nice. Feed it the code and it uses the signature to generate and output a file of passing tests. Then you go and fix the code until the measurements match your expectations. And it could also curve-fit runtime measurements to complexity classes. And derive a set of hypotheses to describe your code by matching the test results against its rule database. And feed the hypotheses to QuickCheck to test them.

It'd work like this:

$ cat foo.ml
let double x = if x = 0 then 0 else 2

$ measure_code < foo.ml
(*
val double : int -> int
Estimated time complexity: O(1) (average runtime 0.01us)
Estimated space complexity: O(1) (allocated an average 8 bytes)
Hypothesis:
double 0 -> 0
forall x:int, double x -> 2
*)
(**T double_against_Int_gen
double 0 = 0
double 1 = 2
double (-1) = 2
double 2 = 2
double 3 = 2
double max_int = 2
double min_int = 2
map double (map (fun i -> 1 lsl i) (0--61)) = replicate 62 2
map double (map (fun i -> -1 lsl i) (0--61)) = replicate 62 2
map double [] = []
**)

And a sketch of the value generator:

module Int_gen =
struct
let zero = 0
let one = 1
let minus_one = -1
let even = 2
let odd = 3
let max_val = max_int
let min_val = min_int
let ever_larger = fun i -> 1 lsl i
let ever_smaller = fun i -> -1 lsl i
let larger_limit = 61
let smaller_limit = 61
let extras = []
end

module Float_gen =
struct
let zero = 0.
let one = 1.
let minus_one = -1.
let even = 2.
let odd = 3.
let max_val = max_float
let min_val = -.max_float
let ever_larger = fun i -> 0.1 *. 2. ** (float i)
let ever_smaller = fun i -> -0.1 *. 2. ** (float i)
let larger_limit = 200
let smaller_limit = 200
let extras = [nan, min_float, -.min_float, infinity, neg_infinity, 1e-320, -1e320]
end

module List_gen (G:GENERATOR) =
struct
let zero = []
let one = [G.one]
let minus_one = [G.minus_one]
let even = [G.zero; G.one]
let odd = [G.minus_one; G.zero; G.one]
let max_val = [G.max_val]
let min_val = [G.min_val]
let ever_larger = fun i -> replicate (1 lsl i) G.one
let ever_smaller = fun i -> []
let larger_limit = 8
let smaller_limit = 0
let extras = [[G.one; G.zero]; [G.max_val; G.min_val]; G.extras]
end
Post a Comment

Blog Archive

About Me

My photo

Built art installations, web sites, graphics libraries, web browsers, mobile apps, desktop apps, media player themes, many nutty prototypes