Return to chapter index

7 Running ScalaCheck

7.1 Retrieving ScalaCheck

name := "MySbtProject"

version := "1.0"

scalaVersion := "2.10.1"

libraryDependencies +=
  "org.scalacheck" %% "scalacheck" % "1.11.0" % "test"


  

7.2 ScalaCheck's testing workflow

7.3 Test parameters

import org.scalacheck.{Prop,Test}

val p = Prop.forAll { xs: List[String] =>
  val s = if(xs.isEmpty) "" else xs.reduce(_+_)
  xs.forall(s.contains)
}

val oneWorker =
  Test.Parameters.default.withMinSuccessfulTests(5000)

val twoWorkers = Test.Parameters.default.
  withMinSuccessfulTests(5000).
  withWorkers(2)


scala> Test.check(oneWorker, p).time
 res0: Long = 4100

scala> Test.check(oneWorker, p).time
 res1: Long = 3982

scala> Test.check(twoWorkers, p).time
 res2: Long = 2565

scala> Test.check(twoWorkers, p).time
 res3: Long = 2577


  

7.4 Integrating ScalaCheck

object Interleaver {
  def interleave[T](l1: List[T], l2: List[T]): List[T] = {
    if(l1.isEmpty) l2
    else if(l2.isEmpty) l1
    else l1.head :: l2.head :: interleave(l2.tail, l1.tail)
  }
}


import Interleaver._
import org.scalatest._
import prop._
import org.scalacheck.Prop.{AnyOperators, forAll, all}

class ExampleSpec extends PropSpec with Checkers {

  property("the interleave method must interleave lists") {
    check(
      forAll { (l1: List[Int], l2: List[Int]) =>
        val res = interleave(l1,l2)
        val is = (0 to Math.min(l1.length,
            l2.length)-1).toList
        all(
          "length" |: l1.length+l2.length =? res.length,
          "zip l1" |: l1 =? is.map(i => res(2*i)) ++
              res.drop(2*l2.length),
          "zip l2" |: l2 =? is.map(i => res(2*i+1)) ++
              res.drop(2*l1.length)
        ) :| ("res: "+res)
      }
    )
  }
}


Run starting. Expected test count is: 1
ExampleSpec:
- the interleave method must interleave lists *** FAILED ***
  GeneratorDrivenPropertyCheckFailedException was thrown
     during property evaluation.
   (c.scala:28)
    Falsified after 3 successful property evaluations.
    Location: (c.scala:28)
    Occurred when passed generated values (
      arg0 = List(0, 0), // 2 shrinks
      arg1 = List(0, 1) // 29 shrinks
    )
    Labels of failing property:
      Expected List("0", "0") but got List("0", "1")
      zip l1
      res: List(0, 0, 1, 0)
Run completed in 241 milliseconds.
Total number of tests run: 1
Suites: completed 1, aborted 0
Tests: succeeded 0, failed 1, canceled 0, ignored 0, pending 0
*** 1 TEST FAILED ***


import Interleaver._
import org.scalatest._
import prop._
import Matchers._

class ExampleSpec extends PropSpec with PropertyChecks {

  property("the interleave method must interleave lists") {
    forAll { (l1: List[Int], l2: List[Int]) =>
      val res = interleave(l1,l2)
      val is = (0 to Math.min(l1.length,
          l2.length)-1).toList
      l1.length + l2.length shouldBe res.length
      l1 shouldBe is.map(i => res(2*i)) ++
          res.drop(2*l2.length)
      l2 shouldBe is.map(i => res(2*i+1)) ++
          res.drop(2*l1.length)
    }
  }
}


Run starting. Expected test count is: 1
ExampleSpec:
- the interleave method must interleave lists *** FAILED ***
  TestFailedException was thrown during property evaluation.
    Message: List(0, 0) did not equal List(0, -1)
    Location: (pc.scala:22)
    Occurred when passed generated values (
      arg0 = List(0, 0), // 2 shrinks
      arg1 = List(0, -1) // 3 shrinks
    )
Run completed in 463 milliseconds.
Total number of tests run: 1
Suites: completed 1, aborted 0
Tests: succeeded 0, failed 1, canceled 0, ignored 0, pending 0
*** 1 TEST FAILED ***


import Interleaver._
import org.specs2._
import org.scalacheck.Prop.{AnyOperators, forAll, all}

class ExampleSpec extends Specification with ScalaCheck {

  def is = s2"""
    the interleave method must interleave lists $e1
  """

  def e1 = {
    forAll { (l1: List[Int], l2: List[Int]) =>
      val res = interleave(l1,l2)
      val is = (0 to Math.min(l1.length,
        l2.length)-1).toList
      all(
        "length" |: l1.length+l2.length =? res.length,
        "zip l1" |: l1 =? is.map(i => res(2*i)) ++
            res.drop(2*l2.length),
        "zip l2" |: l2 =? is.map(i => res(2*i+1)) ++
            res.drop(2*l1.length)
      ) :| ("res: "+res)
    }
  }
}


ExampleSpec
    x the interleave method must interleave lists
 A counter-example is [List(0, 0), List(0, 1)]
     (after 4 tries -
     shrinked ('List(-1, 1, 0, 685354854)' ->
     'List(0, 0)','List(1026517985, -1621352957,
         1638430452)' -> 'List(0, 1)'))
 Expected List("0", "0") but got List("0", "1"),
     zip l1, res: List(0, 0, 1, 0)

Total for specification ExampleSpec
Finished in 24 ms
1 example, 1 failure, 0 error


build.sbt
src/main/scala/MyClass.scala
src/test/scala/MySpec.scala


case class MyClass(s1: String, s2: String) {
  override def toString = "first = "+s1+", second="+s2
}

object MySpec extends org.scalacheck.Properties("MySpec") {
  import org.scalacheck.{Gen, Prop, Arbitrary}
  implicit val arbMyClass = Arbitrary(Gen.resultOf(MyClass))
  property("MyClass.toString") =
    Prop.forAll { mc: MyClass =>
      mc.toString.contains(mc.s1) &&
        mc.toString.contains(mc.s2)
    }
}


$ sbt test
[info] Set current project to MySbtProject (in build ...
[info] Compiling 1 Scala source to /...
[info] + MySpec.MyClass.toString: OK, passed 100 tests.
[info] Passed: : Total 1, Failed 0, Errors 0, Passed 1,
  Skipped 0
[success] Total time: 5 s, completed Sep 10, 2013
  7:55:31 PM


testOptions in Test +=
  Tests.Argument(
    TestFrameworks.ScalaCheck,
    "-maxDiscardRatio", "10",
    "-minSuccessfulTests", "1000"
  )


$ sbt test
[info] Set current project to MySbtProject (in build ...
[info] + MySpec.MyClass.toString: OK, passed 1000 tests.
[info] Passed: : Total 1, Failed 0, Errors 0, Passed 1,
  Skipped 0
[success] Total time: 1 s, completed Sep 11, 2013
  8:14:02 AM


import org.scalacheck.Properties
import org.scalacheck.Prop.forAll

object ListSpec extends Properties("ListSpec") {
  property("append") =
    forAll { (xs: List[Int], n: Int) =>
      (xs :+ n).last == n
    }

  property("insert") =
    forAll { (xs: List[Int], n: Int) =>
      (n +: xs).head == n
    }
}


  $ scalac -cp scalacheck.jar ListSpec.scala
  $ scala -cp scalacheck.jar:. ListSpec
  + ListSpec.append: OK, passed 100 tests.
  + ListSpec.insert: OK, passed 100 tests.


$ scala -cp scalacheck.jar:. ListSpec --help
 Available options:
   -minSize, -n:
      Minimum data generation size
   -maxDiscardRatio, -r:
      The maximum ratio between discarded and succeeded
      tests allowed before ScalaCheck stops testing a
      property. At least minSuccessfulTests will always
      be tested, though.
   -verbosity, -v:
      Verbosity level
   -maxSize, -x:
      Maximum data generation size
   -workers, -w:
      Number of threads to execute in parallel
      for testing
   -minSuccessfulTests, -s:
      Number of tests that must succeed in order to
      pass a property


$ scala -cp scalacheck.jar:. ListSpec -v 2 -s 5000
 + ListSpec.append: OK, passed 5000 tests.
 Elapsed time: 0.374 sec
 + ListSpec.insert: OK, passed 5000 tests.
 Elapsed time: 0.124 sec


case class Result(
  status: Status,
  succeeded: Int,
  discarded: Int,
  freqMap: FreqMap[Set[Any]],
  time: Long
)


import org.scalacheck.Prop.forAll

val singleProp = forAll { (xs: List[Int], n: Int)  =>
  val len = xs.take(n).length
  if (n <= 0) len == 0
  else if (n > xs.length) len == xs.length
  else len == n
}


 scala> import org.scalacheck.Test.{check, Parameters}

 scala> check(Parameters.default, singleProp)
 res0: org.scalacheck.Test.Result =
   Result(Passed,100,0,Map(),9)


import org.scalacheck.Properties

object SomeProps extends Properties("SomeProps") {
  property("add") = forAll { (n:Int, m:Int) => n+m == m+n }
  property("take") = singleProp
}


 scala> import org.scalacheck.Test.checkProperties

 scala> Test.checkProperties(Parameters.default, SomeProps)
 res1: Seq[(String, org.scalacheck.Test.Result)] =
   ListBuffer((SomeProps.add,Result(Passed,100,0,Map(),4)),
     (SomeProps.take,Result(Passed,100,0,Map(),4)))


import org.scalacheck.Prop.forAll
import org.scalacheck.Test.{check, Parameters}

val p = forAll { (s1: String, s2: String) =>
  val s3 = s1+s2
  s3.contains(s1) && s3.contains(s2)
}

val myParams = new Parameters.Default {
  override val minSuccessfulTests = 400
}

check(myParams, p)


val myParams =
  Parameters.default.withMinSuccessfulTests(400)


p.check(_.withMinSuccessfulTests(400))

Test.check(p) { _.
  withMinSuccessfulTests(400).
  withWorkers(4)
}


trait Default extends Parameters {
  val minSuccessfulTests: Int = 100
  val minSize: Int = 0
  val maxSize: Int = Gen.Params().size
  val rng: scala.util.Random = Gen.Params().rng
  val workers: Int = 1
  val testCallback: TestCallback = new TestCallback {}
  val maxDiscardRatio: Float = 5
  val customClassLoader: Option[ClassLoader] = None
}


trait TestCallback {
  /** Called each time a property is evaluated */
  def onPropEval(name: String, threadIdx: Int,
    succeeded: Int, discarded: Int): Unit = ()

  /** Called whenever a property has finished testing */
  def onTestResult(name: String, result:
    Result): Unit = ()
}


import org.scalacheck.Test.{TestCallback, Parameters}
import org.scalacheck.Prop.{forAll, BooleanOperators}

val myTestCallback = new TestCallback {
  override def onPropEval(
    name: String, threadIdx: Int,
    succeeded: Int, discarded: Int
  ): Unit = {
    Console.printf("[%d] %s: s=%d d=%d\n", threadIdx,
      name, succeeded, discarded)
  }
}

val myParams = Parameters.default { _.
  withWorkers(3).
  withTestCallback(myTestCallback)
}

val myProp = forAll { n: Int =>
  (n > 0) ==> (math.abs(n) == n)
}


 scala> org.scalacheck.Test.check(myParams, myProp)
 [2] : s=1 d=0
 [0] : s=1 d=0
 [1] : s=0 d=1
 [0] : s=1 d=1
 ...
 [2] : s=33 d=40
 [2] : s=34 d=40
 res0: org.scalacheck.Test.Result =
   Result(Passed,102,107,Map(),25)


  

7.5 Conclusion