More integration test refactoring

cedar-testing/src/cedar_test_impl.rs

@@ -15,7 +15,9 @@
  */
 
 //! Definition of a `CedarTestImplementation` trait that describes an
-//! implementation of Cedar to use during testing.
+//! implementation of Cedar to use during testing. This trait is used for
+//! running the integration tests and for performing randomized differential
+//! testing (see <https://github.com/cedar-policy/cedar-spec>).
 
 pub use cedar_policy::frontend::is_authorized::InterfaceResponse;
 use cedar_policy_core::ast::{Expr, PolicySet, Request, Value};
@@ -49,6 +51,14 @@ impl<T> TestResult<T> {
             Self::Failure(err) => panic!("{msg}: {err}"),
         }
     }
+
+    /// Apply a function to the success value.
+    pub fn map<F: FnOnce(T) -> T>(self, f: F) -> Self {
+        match self {
+            Self::Success(t) => Self::Success(f(t)),
+            Self::Failure(err) => Self::Failure(err),
+        }
+    }
 }
 
 /// Simple wrapper around u128 to remind ourselves that timing info is in microseconds.
@@ -116,28 +126,44 @@ pub trait CedarTestImplementation {
 
     /// `ErrorComparisonMode` that should be used for this `CedarTestImplementation`
     fn error_comparison_mode(&self) -> ErrorComparisonMode;
+
+    /// `ValidationComparisonMode` that should be used for this `CedarTestImplementation`
+    fn validation_comparison_mode(&self) -> ValidationComparisonMode;
 }
 
-/// Specifies how errors coming from a `CedarTestImplementation` should be
-/// compared against errors coming from the Rust implementation.
+/// Specifies how authorization errors coming from this [`CedarTestImplementation`]
+///  should be compared against errors coming from another implementation.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub enum ErrorComparisonMode {
-    /// Don't compare errors at all; the `CedarTestImplementation` is not
-    /// expected to produce errors matching the Rust implementation's errors in
-    /// any way.
-    /// In fact, the `CedarTestImplementation` will be expected to never report
-    /// errors.
+    /// Don't compare errors at all. The [`CedarTestImplementation`] will be
+    /// expected to never report errors.
     Ignore,
-    /// The `CedarTestImplementation` is expected to produce "error messages" that
-    /// are actually just the id of the erroring policy. This will be compared to
-    /// ensure that the `CedarTestImplementation` agrees with the Rust
-    /// implementation on which policies produce errors.
+    /// The [`CedarTestImplementation`] is expected to produce "error messages"
+    /// that are actually just the id of the erroring policy. This will used to
+    /// ensure that different implementations agree on which policies produce
+    /// errors.
     PolicyIds,
-    /// The `CedarTestImplementation` is expected to produce error messages that
+    /// The [`CedarTestImplementation`] is expected to produce error messages that
     /// exactly match the Rust implementation's error messages' `Display` text.
     Full,
 }
 
+/// Specifies how validation results from this [`CedarTestImplementation`] should
+/// be compared against validation results from another implementation.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum ValidationComparisonMode {
+    /// When comparing this [`CedarTestImplementation`] against another
+    /// implementation, `validate` should return a `validation_passed` result
+    /// for any input that the other implementation says is valid. This allows
+    /// for flexibility in cases where the other implementation (incorrectly)
+    /// says the input is invalid due to weaker typing precision.
+    AgreeOnValid,
+    /// When comparing this [`CedarTestImplementation`] against another
+    /// implementation, the valid / not valid decision should agree for all
+    /// inputs, although the exact validation errors may differ.
+    AgreeOnAll,
+}
+
 /// Basic struct to support implementing the `CedarTestImplementation` trait
 #[derive(Debug, Default)]
 pub struct RustEngine {}
@@ -231,4 +257,8 @@ impl CedarTestImplementation for RustEngine {
     fn error_comparison_mode(&self) -> ErrorComparisonMode {
         ErrorComparisonMode::PolicyIds
     }
+
+    fn validation_comparison_mode(&self) -> ValidationComparisonMode {
+        ValidationComparisonMode::AgreeOnAll
+    }
 }

cedar-testing/src/integration_testing.rs

@@ -313,10 +313,15 @@ pub fn perform_integration_test_from_json_custom(
             validation_result.errors
         );
     } else {
-        assert!(
-            !validation_result.validation_passed(),
-            "Expected that validation would fail in {test_name}, but it did not.",
-        );
+        match test_impl.validation_comparison_mode() {
+            ValidationComparisonMode::AgreeOnAll => {
+                assert!(
+                    !validation_result.validation_passed(),
+                    "Expected that validation would fail in {test_name}, but it did not.",
+                );
+            }
+            ValidationComparisonMode::AgreeOnValid => {} // ignore
+        }
     }
 
     for json_request in test.requests {

cedar-testing/tests/cedar-policy-cli/main.rs

@@ -20,9 +20,12 @@
 #![allow(clippy::expect_used)]
 // PANIC SAFETY tests
 #![allow(clippy::panic)]
+
 mod corpus_tests;
+#[cfg(feature = "decimal")]
 mod decimal;
 mod example_use_cases;
+#[cfg(feature = "ipaddr")]
 mod ip;
 mod multi;
 

cedar-testing/tests/cedar-policy/corpus_tests.rs

@@ -16,8 +16,8 @@
 
 //! Integration tests auto-generated using the differential tester.
 
-use crate::integration_testing::perform_integration_test_from_json;
-use crate::integration_testing::resolve_integration_test_path;
+use cedar_testing::integration_testing::perform_integration_test_from_json;
+use cedar_testing::integration_testing::resolve_integration_test_path;
 use std::path::Path;
 
 /// Path of the folder containing the corpus tests

cedar-testing/tests/cedar-policy/decimal.rs

@@ -16,7 +16,7 @@
 
 //! Integration tests targeting the decimal extension
 
-use crate::integration_testing::perform_integration_test_from_json;
+use cedar_testing::integration_testing::perform_integration_test_from_json;
 use std::path::Path;
 
 /// Path of the folder containing the JSON tests
@@ -25,13 +25,11 @@ fn folder() -> &'static Path {
 }
 
 #[test]
-#[cfg(feature = "decimal")]
 fn decimal_1() {
     perform_integration_test_from_json(folder().join("1.json"));
 }
 
 #[test]
-#[cfg(feature = "decimal")]
 fn decimal_2() {
     perform_integration_test_from_json(folder().join("2.json"));
 }

cedar-testing/tests/cedar-policy/example_use_cases.rs

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-use crate::integration_testing::perform_integration_test_from_json;
+use cedar_testing::integration_testing::perform_integration_test_from_json;
 use std::path::Path;
 
 /// Path of the folder containing the JSON tests

cedar-testing/tests/cedar-policy/ip.rs

@@ -16,7 +16,7 @@
 
 //! Integration tests targeting the ipaddr extension
 
-use crate::integration_testing::perform_integration_test_from_json;
+use cedar_testing::integration_testing::perform_integration_test_from_json;
 use std::path::Path;
 
 /// Path of the folder containing the JSON tests
@@ -25,19 +25,16 @@ fn folder() -> &'static Path {
 }
 
 #[test]
-#[cfg(feature = "ipaddr")]
 fn ip_1() {
     perform_integration_test_from_json(folder().join("1.json"));
 }
 
 #[test]
-#[cfg(feature = "ipaddr")]
 fn ip_2() {
     perform_integration_test_from_json(folder().join("2.json"));
 }
 
 #[test]
-#[cfg(feature = "ipaddr")]
 fn ip_3() {
     perform_integration_test_from_json(folder().join("3.json"));
 }

cedar-testing/tests/cedar-policy/main.rs

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2022-2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// PANIC SAFETY tests
+#![allow(clippy::expect_used)]
+// PANIC SAFETY tests
+#![allow(clippy::panic)]
+
+mod corpus_tests;
+#[cfg(feature = "decimal")]
+mod decimal;
+mod example_use_cases;
+#[cfg(feature = "ipaddr")]
+mod ip;
+mod multi;

cedar-testing/tests/cedar-policy/multi.rs

@@ -16,7 +16,7 @@
 
 //! Integration tests that involve interactions between multiple policies
 
-use crate::integration_testing::perform_integration_test_from_json;
+use cedar_testing::integration_testing::perform_integration_test_from_json;
 use std::path::Path;
 
 /// Path of the folder containing the JSON tests