art with code

2009-01-04

Random thought on automated testing

An automatic test generator that takes the function signature would be nice. Feed it the code and it uses the signature to generate and output a file of passing tests. Then you go and fix the code until the measurements match your expectations. And it could also curve-fit runtime measurements to complexity classes. And derive a set of hypotheses to describe your code by matching the test results against its rule database. And feed the hypotheses to QuickCheck to test them.

It'd work like this:

$ cat foo.ml
let double x = if x = 0 then 0 else 2

$ measure_code < foo.ml
(*
val double : int -> int
Estimated time complexity: O(1) (average runtime 0.01us)
Estimated space complexity: O(1) (allocated an average 8 bytes)
Hypothesis:
double 0 -> 0
forall x:int, double x -> 2
*)
(**T double_against_Int_gen
double 0 = 0
double 1 = 2
double (-1) = 2
double 2 = 2
double 3 = 2
double max_int = 2
double min_int = 2
map double (map (fun i -> 1 lsl i) (0--61)) = replicate 62 2
map double (map (fun i -> -1 lsl i) (0--61)) = replicate 62 2
map double [] = []
**)

And a sketch of the value generator:

module Int_gen =
struct
let zero = 0
let one = 1
let minus_one = -1
let even = 2
let odd = 3
let max_val = max_int
let min_val = min_int
let ever_larger = fun i -> 1 lsl i
let ever_smaller = fun i -> -1 lsl i
let larger_limit = 61
let smaller_limit = 61
let extras = []
end

module Float_gen =
struct
let zero = 0.
let one = 1.
let minus_one = -1.
let even = 2.
let odd = 3.
let max_val = max_float
let min_val = -.max_float
let ever_larger = fun i -> 0.1 *. 2. ** (float i)
let ever_smaller = fun i -> -0.1 *. 2. ** (float i)
let larger_limit = 200
let smaller_limit = 200
let extras = [nan, min_float, -.min_float, infinity, neg_infinity, 1e-320, -1e320]
end

module List_gen (G:GENERATOR) =
struct
let zero = []
let one = [G.one]
let minus_one = [G.minus_one]
let even = [G.zero; G.one]
let odd = [G.minus_one; G.zero; G.one]
let max_val = [G.max_val]
let min_val = [G.min_val]
let ever_larger = fun i -> replicate (1 lsl i) G.one
let ever_smaller = fun i -> []
let larger_limit = 8
let smaller_limit = 0
let extras = [[G.one; G.zero]; [G.max_val; G.min_val]; G.extras]
end
Post a Comment

Blog Archive

About Me

My photo

Built art installations, web sites, graphics libraries, web browsers, mobile apps, desktop apps, media player themes, many nutty prototypes, much bad code, much bad art.

Have freelanced for Verizon, Google, Mozilla, Warner Bros, Sony Pictures, Yahoo!, Microsoft, Valve Software, TDK Electronics.

Ex-Chrome Developer Relations.