Adds contraint tests for cpp.

evaluate_genericify_cpp.py

@@ -167,8 +167,11 @@ class EvaluateGenericifyCpp(evaluate.Metric):
             result.sort()
             for pt in [
                 "base_run_passed",
+                "base_run_compiled",
                 "sfinae_run_passed",
+                "sfinae_run_compiled",
                 "concepts_run_passed",
+                "concepts_run_compiled",
             ]:
                 passed = [r[1][pt] for r in result]
                 totals[pt].append(len(passed))

execute.py

@@ -21,6 +21,7 @@ def check_correctness(candidate, reference, task_id, completion_id):
         candidate["base"],
         reference["tests"],
         base_run_result,
+        "c++17",
     )
     sfinae_run_result = manager.list()
     process_case(
@@ -28,6 +29,7 @@ def check_correctness(candidate, reference, task_id, completion_id):
         candidate["sfinae"],
         reference["tests"],
         sfinae_run_result,
+        "c++17",
     )
     concepts_run_result = manager.list()
     process_case(
@@ -35,26 +37,60 @@ def check_correctness(candidate, reference, task_id, completion_id):
         candidate["concepts"],
         reference["tests"],
         concepts_run_result,
+        "c++20",
+    )
+
+    sfinae_constrain_result = manager.list()
+    process_case(
+        invalid_compile_cpp,
+        candidate["sfinae"],
+        reference["invalids"],
+        sfinae_constrain_result,
+        "c++17",
+    )
+    concepts_constrain_result = manager.list()
+    process_case(
+        invalid_compile_cpp,
+        candidate["concepts"],
+        reference["invalids"],
+        concepts_constrain_result,
+        "c++20",
     )
 
     return dict(
         task_id=task_id,
         completion_id=completion_id,
         base_run_passed=base_run_result[0] == "passed",
+        base_run_compiled=(
+            base_run_result[0] == "passed"
+            or base_run_result[0].startswith("failed: runtime error:")
+        ),
         base_run_result=base_run_result[0],
         sfinae_run_passed=sfinae_run_result[0] == "passed",
+        sfinae_run_compiled=(
+            sfinae_run_result[0] == "passed"
+            or sfinae_run_result[0].startswith("failed: runtime error:")
+        ),
         sfinae_run_result=sfinae_run_result[0],
         concepts_run_passed=concepts_run_result[0] == "passed",
+        concepts_run_compiled=(
+            concepts_run_result[0] == "passed"
+            or concepts_run_result[0].startswith("failed: runtime error:")
+        ),
         concepts_run_result=concepts_run_result[0],
+        sfinae_constrain_passed=sfinae_constrain_result[0] == "passed",
+        sfinae_constrain_result=sfinae_constrain_result[0],
+        concepts_constrain_passed=concepts_constrain_result[0] == "passed",
+        concepts_constrain_result=concepts_constrain_result[0],
     )
 
 
-def process_case(target, candidate, reference, result):
+def process_case(target, candidate, reference, result, cppstd):
     timeout = 60
 
     p = multiprocessing.Process(
         target=target,
-        args=(candidate, reference, result, timeout),
+        args=(candidate, reference, result, timeout, cppstd),
     )
 
     p.start()
@@ -66,14 +102,14 @@ def process_case(target, candidate, reference, result):
         result.append("timed out")
 
 
-def unsafe_execute_cpp(candidate, reference, result, timeout):
+def unsafe_execute_cpp(candidate, reference, result, timeout, cppstd):
     with create_tempdir():
         code = "#include <bits/stdc++.h>\n" + candidate + reference
         open(f"test.cpp", "w").write(code)
 
         cpp_compiler = os.getenv("GENERICIFY_CLANG")
         compilation_result = subprocess.run(
-            [cpp_compiler, "-std=c++20", "test.cpp"],
+            [cpp_compiler, f"-std={cppstd}", "test.cpp"],
             timeout=timeout,
             capture_output=True,
         )
@@ -102,12 +138,38 @@ def unsafe_execute_cpp(candidate, reference, result, timeout):
                             err = exec_result.stdout.decode()
                         except:
                             err = exec_result.stdout
-                    result.append(f"failed: {err}")
+                    result.append(f"failed: runtime error: {err}")
 
             except subprocess.TimeoutExpired as e:
                 result.append("timed out")
 
 
+def invalid_compile_cpp(candidate, reference, result, timeout, cppstd):
+    with create_tempdir():
+        code = "#include <bits/stdc++.h>\n" + candidate + reference
+        open(f"invalid.cpp", "w").write(code)
+
+        cpp_compiler = os.getenv("GENERICIFY_CLANG")
+        compilation_result = subprocess.run(
+            [cpp_compiler, f"-std={cppstd}", "invalid.cpp"],
+            timeout=timeout,
+            capture_output=True,
+        )
+
+        if compilation_result.stderr:
+            err = compilation_result.stderr.decode()
+        else:
+            err = compilation_result.stdout.decode()
+
+        if compilation_result.returncode != 1:
+            result.append(f"failed: compilation succeeded: {err}")
+        else:
+            if "note: candidate template ignored" in err:
+                result.append("passed")
+            else:
+                result.append("failed: improperly constrained: {err}")
+
+
 @contextlib.contextmanager
 def create_tempdir():
     with tempfile.TemporaryDirectory() as dirname:

tests.py

@@ -1,17 +1,89 @@
 test_cases = [
     {
-        "predictions": [0, 0],
-        "references": [1, 1],
-        "result": {"metric_score": 0}
+        "predictions": [
+            [
+                {
+                    "base": r"""
+template <typename T>
+bool has_close_elements(T numbers, float threshold) {
+  for (int i = 0; i < numbers.size(); i++)
+    for (int j = i + 1; j < numbers.size(); j++)
+      if (std::abs(numbers[i] - numbers[j]) < threshold)
+        return true;
+
+  return false;
+}
+""",
+                    "sfinae": r"""
+template <
+    typename T,
+    std::enable_if_t<std::is_same_v<typename T::value_type, float>, int> = 0>
+bool has_close_elements(T numbers, float threshold) {
+  for (int i = 0; i < numbers.size(); i++)
+    for (int j = i + 1; j < numbers.size(); j++)
+      if (std::abs(numbers[i] - numbers[j]) < threshold)
+        return true;
+
+  return false;
+}
+""",
+                    "concepts": r"""
+template <typename T>
+  requires std::same_as<typename T::value_type, float>
+bool has_close_elements(T numbers, float threshold) {
+  for (int i = 0; i < numbers.size(); i++)
+    for (int j = i + 1; j < numbers.size(); j++)
+      if (std::abs(numbers[i] - numbers[j]) < threshold)
+        return true;
+
+  return false;
+}
+""",
+                }
+            ]
+        ],
+        "references": [
+            {
+                "tests": r"""
+#define ASSERT(...)                                                            \
+  do {                                                                         \
+    if (!(__VA_ARGS__)) {                                                      \
+      std::exit(-1);                                                           \
+    }                                                                          \
+  } while (false)
+
+#define TEST_ON_TYPE(_type_)                                                   \
+  do {                                                                         \
+    _type_ a = {1.0, 2.0, 3.9, 4.0, 5.0, 2.2};                                 \
+    ASSERT(has_close_elements(a, 0.3) == true);                                \
+    ASSERT(has_close_elements(a, 0.05) == false);                              \
+    ASSERT(has_close_elements(_type_{1.0, 2.0, 5.9, 4.0, 5.0}, 0.95) == true); \
+    ASSERT(has_close_elements(_type_{1.0, 2.0, 5.9, 4.0, 5.0}, 0.8) == false); \
+    ASSERT(has_close_elements(_type_{1.0, 2.0, 3.0, 4.0, 5.0}, 2.0) == true);  \
+    ASSERT(has_close_elements(_type_{1.1, 2.2, 3.1, 4.1, 5.1}, 1.0) == true);  \
+    ASSERT(has_close_elements(_type_{1.1, 2.2, 3.1, 4.1, 5.1}, 0.5) == false); \
+  } while (false)
+
+int main() {
+  TEST_ON_TYPE(std::vector<float>);
+  TEST_ON_TYPE(std::deque<float>);
+}
+""",
+                "invalids": r"""
+int main() {
+  std::string s{};
+  has_close_elements(s, 3.4);
+}
+""",
+            }
+        ],
+        "result": {
+            "base_run_passed@1": 1.0,
+            "base_run_compiled@1": 1.0,
+            "sfinae_run_passed@1": 1.0,
+            "sfinae_run_compiled@1": 1.0,
+            "concepts_run_passed@1": 1.0,
+            "concepts_run_compiled@1": 1.0,
+        },
     },
-    {
-        "predictions": [1, 1],
-        "references": [1, 1],
-        "result": {"metric_score": 1}
-    },
-    {
-        "predictions": [1, 0],
-        "references": [1, 1],
-        "result": {"metric_score": 0.5}
-    }
-]
+]