More integration test refactoring

cedar-testing/src/cedar_test_impl.rs

@@ -15,7 +15,9 @@
  */
 
 //! Definition of a `CedarTestImplementation` trait that describes an
-//! implementation of Cedar to use during testing.
+//! implementation of Cedar to use during testing. This trait is used for
+//! running the integration tests and for performing randomized differential
+//! testing (see <https://github.com/cedar-policy/cedar-spec>).
 
 pub use cedar_policy::frontend::is_authorized::InterfaceResponse;
 use cedar_policy_core::ast::{Expr, PolicySet, Request, Value};
@@ -116,28 +118,43 @@ pub trait CedarTestImplementation {
 
     /// `ErrorComparisonMode` that should be used for this `CedarTestImplementation`
     fn error_comparison_mode(&self) -> ErrorComparisonMode;
+
+    /// `ValidationComparisonMode` that should be used for this `CedarTestImplementation`
+    fn validation_comparison_mode(&self) -> ValidationComparisonMode;
 }
 
-/// Specifies how errors coming from a `CedarTestImplementation` should be
-/// compared against errors coming from the Rust implementation.
+/// Specifies how authorization errors coming from this `CedarTestImplementation`
+///  should be compared against errors coming from another implementation.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub enum ErrorComparisonMode {
-    /// Don't compare errors at all; the `CedarTestImplementation` is not
-    /// expected to produce errors matching the Rust implementation's errors in
-    /// any way.
-    /// In fact, the `CedarTestImplementation` will be expected to never report
-    /// errors.
+    /// Don't compare errors at all.
     Ignore,
-    /// The `CedarTestImplementation` is expected to produce "error messages" that
-    /// are actually just the id of the erroring policy. This will be compared to
-    /// ensure that the `CedarTestImplementation` agrees with the Rust
-    /// implementation on which policies produce errors.
+    /// The `CedarTestImplementation` is expected to produce "error messages"
+    /// that are actually just the id of the erroring policy. This will used to
+    /// ensure that different `CedarTestImplementation`s agree on which policies
+    /// produce errors.
     PolicyIds,
     /// The `CedarTestImplementation` is expected to produce error messages that
     /// exactly match the Rust implementation's error messages' `Display` text.
     Full,
 }
 
+/// Specifies how validation results from this `CedarTestImplementation` should
+/// compared against validation results from another implementation.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub enum ValidationComparisonMode {
+    /// When comparing this `CedarTestImplementation` against another
+    /// implementation, `validate` should return a `validation_passed` result
+    /// for any input that the other implementation says is valid. This allows
+    /// for flexibility in cases where the other implementation (incorrectly)
+    /// says the input is invalid due to weaker typing precision.
+    AgreeOnValid,
+    /// When comparing this `CedarTestImplementation` against another
+    /// implementation, the valid / not valid decision should agree for all
+    /// inputs, although the exact validation errors may differ.
+    AgreeOnAll,
+}
+
 /// Basic struct to support implementing the `CedarTestImplementation` trait
 #[derive(Debug, Default)]
 pub struct RustEngine {}
@@ -231,4 +248,8 @@ impl CedarTestImplementation for RustEngine {
     fn error_comparison_mode(&self) -> ErrorComparisonMode {
         ErrorComparisonMode::PolicyIds
     }
+
+    fn validation_comparison_mode(&self) -> ValidationComparisonMode {
+        ValidationComparisonMode::AgreeOnAll
+    }
 }

cedar-testing/src/integration_testing.rs

@@ -312,7 +312,13 @@ pub fn perform_integration_test_from_json_custom(
             "Unexpected validation errors in {test_name}: {:?}",
             validation_result.errors
         );
-    } else {
+    }
+    if !test.should_validate
+        && matches!(
+            test_impl.validation_comparison_mode(),
+            ValidationComparisonMode::AgreeOnAll
+        )
+    {
         assert!(
             !validation_result.validation_passed(),
             "Expected that validation would fail in {test_name}, but it did not.",

cedar-testing/tests/cedar-policy/corpus_tests.rs

@@ -16,8 +16,8 @@
 
 //! Integration tests auto-generated using the differential tester.
 
-use crate::integration_testing::perform_integration_test_from_json;
-use crate::integration_testing::resolve_integration_test_path;
+use cedar_testing::integration_testing::perform_integration_test_from_json;
+use cedar_testing::integration_testing::resolve_integration_test_path;
 use std::path::Path;
 
 /// Path of the folder containing the corpus tests

cedar-testing/tests/cedar-policy/decimal.rs

@@ -16,7 +16,7 @@
 
 //! Integration tests targeting the decimal extension
 
-use crate::integration_testing::perform_integration_test_from_json;
+use cedar_testing::integration_testing::perform_integration_test_from_json;
 use std::path::Path;
 
 /// Path of the folder containing the JSON tests

cedar-testing/tests/cedar-policy/example_use_cases.rs

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-use crate::integration_testing::perform_integration_test_from_json;
+use cedar_testing::integration_testing::perform_integration_test_from_json;
 use std::path::Path;
 
 /// Path of the folder containing the JSON tests

cedar-testing/tests/cedar-policy/ip.rs

@@ -16,7 +16,7 @@
 
 //! Integration tests targeting the ipaddr extension
 
-use crate::integration_testing::perform_integration_test_from_json;
+use cedar_testing::integration_testing::perform_integration_test_from_json;
 use std::path::Path;
 
 /// Path of the folder containing the JSON tests

cedar-testing/tests/cedar-policy/main.rs

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2022-2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// PANIC SAFETY tests
+#![allow(clippy::expect_used)]
+// PANIC SAFETY tests
+#![allow(clippy::panic)]
+mod corpus_tests;
+mod decimal;
+mod example_use_cases;
+mod ip;
+mod multi;

cedar-testing/tests/cedar-policy/multi.rs

@@ -16,7 +16,7 @@
 
 //! Integration tests that involve interactions between multiple policies
 
-use crate::integration_testing::perform_integration_test_from_json;
+use cedar_testing::integration_testing::perform_integration_test_from_json;
 use std::path::Path;
 
 /// Path of the folder containing the JSON tests