junior-dev/gcd.yaml

.Checks: &Checks
    one_argument:
      assert: "len(f.args)"
      eq: 2
    gcd_1_1:
      assert: "f.call(1,1)"
      eq: 1
    gcd_100_10:
      assert: "f.call(100,10)"
      eq: 10

GCDZeroShot:
    Signature: "gcd(a, b)"
    Input: "with two inputs a and b"
    Output: "the greatest common demoninator (gcd) of a and b"
    Description: "See if the model can output a well-known, simple algorithm."
    Checks:
        <<: *Checks
        input_name_0:
            assert: "f.args[0].name"
            eq: "a"
        input_name_1:
            assert: "f.args[1].name"
            eq: "b"

GCDMisnamed:
    Signature: "fofx(x, banana)"
    Input: "with two inputs x and banana"
    Output: "the greatest common demoninator (gcd) of x and banana"
    Description: "See if the model can output a well-known, simple algorithm if the name has been changed."
    Checks:
        <<: *Checks
        input_name_0:
            assert: "f.args[0].name"
            eq: "x"
        input_name_1:
            assert: "f.args[1].name"
            eq: "banana"
        func_name:
            assert: "f.name"
            eq: "fofx"