junior-dev/sanity.yaml

SanityMeaningOfLife:
    Signature: "meaning_of_life()"
    Input: "with no inputs"
    Output: "a single integer, the answer to life the universe and everything"
    Fact: "the answer to life the universe and everything is 42"
    Description: "Scalar function with no arguments, see if the model can use its internal knowledge to both code and summon a fact from pop culture."
    Checks:
        input_name:
            assert: "f.name"
            eq: "meaning_of_life"
        no_arguments:
          assert: "len(f.args)"
          eq: 0
        h2g2:
          assert: "f.call()"
          eq: 42

SanityList:
    Signature: "things()"
    Input: "with no inputs"
    Output: "a list with three values: the number 5, the string 'foobar', the capital city of Spain"
    Fact: "the capital city of Spain is Madrid"
    Description: "List function, see if the model can combine input facts with internal knowledge."
    Checks:
        input_name:
            assert: "f.name"
            eq: "things"
        no_arguments:
          assert: "len(f.args)"
          eq: 0
        returns:
          assert: "f.call()"
          eq: [5, "foobar", "Madrid"]

SanitySecretIdentityMap:
    Signature: "secretidentities()"
    Input: "with no inputs"
    Output: "an object mapping the superheroes Superman, Batman and Spider-Man to their real names"
    Fact: "Superman's real name is Clark Kent. Batman's real name is Bruce Wayne. Spider-Man's real name is Peter Parker."
    Description: "Object function, see if the model can use its internal knowledge to build a map."
    Checks:
        input_name:
            assert: "f.name"
            eq: "secretidentities"
        no_arguments:
          assert: "len(f.args)"
          eq: 0
        superman:
          assert: "f.call().get('Superman')"
          eq: "Clark Kent"
        batman:
          assert: "f.call().get('Batman')"
          eq: "Bruce Wayne"
        spiderman:
          assert: "f.call().get('Spider-Man')"
          eq: "Peter Parker"