Skip to content

validation - index only fields found within filter function #2969

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: integration
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,17 @@
* A {@link QueryRule} implementation that will check if any indexed fields are used within the functions {@code filter:includeRegex} or
* {@code filter:excludeRegex} in a query.
*/
public class IncludeExcludeIndexFieldsRule extends ShardQueryRule {
public class IncludeExcludeIndexOnlyFieldsRule extends ShardQueryRule {

private static final Logger log = Logger.getLogger(IncludeExcludeIndexFieldsRule.class);
private static final Logger log = Logger.getLogger(IncludeExcludeIndexOnlyFieldsRule.class);

private static final Set<Pair<String,String>> functions = Collections.unmodifiableSet(Sets.newHashSet(
Pair.of(EvaluationPhaseFilterFunctions.EVAL_PHASE_FUNCTION_NAMESPACE, EvaluationPhaseFilterFunctionsDescriptor.INCLUDE_REGEX),
Pair.of(EvaluationPhaseFilterFunctions.EVAL_PHASE_FUNCTION_NAMESPACE, EvaluationPhaseFilterFunctionsDescriptor.EXCLUDE_REGEX)));

public IncludeExcludeIndexFieldsRule() {}
public IncludeExcludeIndexOnlyFieldsRule() {}

public IncludeExcludeIndexFieldsRule(String name) {
public IncludeExcludeIndexOnlyFieldsRule(String name) {
super(name);
}

Expand All @@ -50,16 +50,16 @@ public QueryRuleResult validate(QueryValidationConfiguration ruleConfiguration)
ASTJexlScript jexlScript = (ASTJexlScript) ruleConfig.getParsedQuery();
// Fetch the set of fields given within any filter:includeRegex or filter:excludeRegex function calls in the query, if any.
Set<FetchFunctionFieldsVisitor.FunctionFields> functions = FetchFunctionFieldsVisitor.fetchFields(jexlScript,
IncludeExcludeIndexFieldsRule.functions, metadataHelper);
IncludeExcludeIndexOnlyFieldsRule.functions, metadataHelper);
if (!functions.isEmpty()) {
Set<String> indexedFields = metadataHelper.getIndexedFields(null);
Set<String> indexOnlyFields = metadataHelper.getIndexOnlyFields(null);
// Each FunctionField object represents the collection of all fields seen for either filter:includeRegex or filter:excludeRegex.
for (FetchFunctionFieldsVisitor.FunctionFields functionFields : functions) {
Set<String> intersection = Sets.intersection(indexedFields, functionFields.getFields());
// If the function contains any index fields, add a message to the result.
Set<String> intersection = Sets.intersection(indexOnlyFields, functionFields.getFields());
// If the function contains any index only fields, add a message to the result.
if (!intersection.isEmpty()) {
result.addMessage("Indexed fields found within the function " + functionFields.getNamespace() + ":" + functionFields.getFunction()
+ ": " + String.join(", ", intersection));
result.addMessage("Index Only fields found within the filter function " + functionFields.getNamespace() + ":"
+ functionFields.getFunction() + ": " + String.join(", ", intersection));
}
}
}
Expand All @@ -73,6 +73,6 @@ public QueryRuleResult validate(QueryValidationConfiguration ruleConfiguration)

@Override
public QueryRule copy() {
return new IncludeExcludeIndexFieldsRule(name);
return new IncludeExcludeIndexOnlyFieldsRule(name);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@

import datawave.query.util.MockMetadataHelper;

class IncludeExcludeIndexFieldsRuleTest extends ShardQueryRuleTest {
class IncludeExcludeIndexOnlyFieldsRuleTest extends ShardQueryRuleTest {

private static final MockMetadataHelper metadataHelper = new MockMetadataHelper();

@BeforeAll
static void beforeAll() {
metadataHelper.setIndexedFields(Set.of("INDEXED1", "INDEXED2"));
metadataHelper.setIndexOnlyFields(Set.of("INDEXED1", "INDEXED2"));
}

@BeforeEach
Expand All @@ -38,31 +38,31 @@ void testQueryWithoutFunctions() throws Exception {
}

/**
* Test versions of the includeRegex and excludeRegex functions without indexed fields.
* Test versions of the includeRegex and excludeRegex functions without index only fields.
*
* @param name
* the function name
*/
@ParameterizedTest
@ValueSource(strings = {"includeRegex", "excludeRegex"})
void testFunctionWithoutIndexedField(String name) throws Exception {
void testFunctionWithoutIndexedOnlyField(String name) throws Exception {
givenQuery("filter:" + name + "(FOO,'value')");

// Do not expect any messages.
assertResult();
}

/**
* Test versions of the includeRegex and excludeRegex functions with an indexed field.
* Test versions of the includeRegex and excludeRegex functions with an index only field.
*
* @param name
* the function name
*/
@ParameterizedTest
@ValueSource(strings = {"includeRegex", "excludeRegex"})
void testFunctionWithSingleIndexedField(String name) throws Exception {
void testFunctionWithSingleIndexOnlyField(String name) throws Exception {
givenQuery("filter:" + name + "(INDEXED1,'value')");
expectMessage("Indexed fields found within the function filter:" + name + ": INDEXED1");
expectMessage("Index Only fields found within the filter function filter:" + name + ": INDEXED1");

assertResult();
}
Expand All @@ -71,10 +71,10 @@ void testFunctionWithSingleIndexedField(String name) throws Exception {
* Test a query with both the includeRegex and excludeRegex functions with indexed fields.
*/
@Test
void testMultipleFunctionWithIndexedField() throws Exception {
void testMultipleFunctionWithIndexOnlyField() throws Exception {
givenQuery("filter:includeRegex(INDEXED1,'value') && filter:excludeRegex(INDEXED2, 'value')");
expectMessage("Indexed fields found within the function filter:includeRegex: INDEXED1");
expectMessage("Indexed fields found within the function filter:excludeRegex: INDEXED2");
expectMessage("Index Only fields found within the filter function filter:includeRegex: INDEXED1");
expectMessage("Index Only fields found within the filter function filter:excludeRegex: INDEXED2");

assertResult();
}
Expand All @@ -83,7 +83,7 @@ void testMultipleFunctionWithIndexedField() throws Exception {
* Test a query with both the includeRegex and excludeRegex functions without indexed fields.
*/
@Test
void testMultipleFunctionWithoutIndexedField() throws Exception {
void testMultipleFunctionWithoutIndexOnlyField() throws Exception {
givenQuery("filter:includeRegex(FOO,'value') && filter:excludeRegex(BAR, 'value')");

// Do not expect any messages.
Expand All @@ -97,6 +97,6 @@ protected Object parseQuery() throws Exception {

@Override
protected ShardQueryRule getNewRule() {
return new IncludeExcludeIndexFieldsRule(ruleName);
return new IncludeExcludeIndexOnlyFieldsRule(ruleName);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
import datawave.query.rules.FieldExistenceRule;
import datawave.query.rules.FieldPatternPresenceRule;
import datawave.query.rules.IncludeExcludeArgsRule;
import datawave.query.rules.IncludeExcludeIndexFieldsRule;
import datawave.query.rules.IncludeExcludeIndexOnlyFieldsRule;
import datawave.query.rules.InvalidQuoteRule;
import datawave.query.rules.MinimumSlopProximityRule;
import datawave.query.rules.NumericValueRule;
Expand Down Expand Up @@ -172,7 +172,7 @@ public void testBeanCreation() {
expectedRules.add(new UnescapedSpecialCharsRule("Check for Unescaped Special Characters", Set.of('?'), Set.of('_'), false, false));
expectedRules.add(new FieldPatternPresenceRule("Check Presence of Field or Pattern", Map.of("_ANYFIELD_", "Detected presence of _ANYFIELD_"),
Map.of(".*", "Detected pattern '.*' that will match everything")));
expectedRules.add(new IncludeExcludeIndexFieldsRule("Check #INCLUDE and #EXCLUDE for Indexed Fields"));
expectedRules.add(new IncludeExcludeIndexOnlyFieldsRule("Check #INCLUDE and #EXCLUDE for Index Only Fields"));
expectedRules.add(new NumericValueRule("Validate Numeric Values Only Given for Numeric Fields"));
expectedRules.add(new TimeFunctionRule("Validate #TIME_FUNCTION has Date Fields"));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,8 +188,8 @@
<property name="name" value="Validate Args of #INCLUDE and #EXCLUDE"/>
</bean>

<bean id="IncludeExcludeIndexFieldsRule" scope="prototype" class="datawave.query.rules.IncludeExcludeIndexFieldsRule">
<property name="name" value="Check #INCLUDE and #EXCLUDE for Indexed Fields"/>
<bean id="IncludeExcludeIndexOnlyFieldsRule" scope="prototype" class="datawave.query.rules.IncludeExcludeIndexOnlyFieldsRule">
<property name="name" value="Check #INCLUDE and #EXCLUDE for Index Only Fields"/>
</bean>

<bean id="InvalidQuoteRule" scope="prototype" class="datawave.query.rules.InvalidQuoteRule">
Expand Down Expand Up @@ -404,7 +404,7 @@
<ref bean="FieldExistenceRule"/>
<ref bean="UnescapedSpecialCharsRule"/>
<ref bean="FieldPatternPresenceRule"/>
<ref bean="IncludeExcludeIndexFieldsRule"/>
<ref bean="IncludeExcludeIndexOnlyFieldsRule"/>
<ref bean="NumericValueRule"/>
<ref bean="TimeFunctionRule"/>
</list>
Expand Down