IBX-9845: Upgraded Solr to version 9.8.1

.github/init_solr.sh

@@ -1,8 +1,18 @@
 #!/usr/bin/env bash
 
-default_config_files[1]='src/lib/Resources/config/solr/schema.xml'
-default_config_files[2]='src/lib/Resources/config/solr/custom-fields-types.xml'
-default_config_files[3]='src/lib/Resources/config/solr/language-fieldtypes.xml'
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+SOLR_VERSION=${SOLR_VERSION:-'9.8.1'}
+
+if [[ "${SOLR_VERSION}" =~ ^9\. ]]; then
+    default_config_files[1]="${SCRIPT_DIR}/../src/lib/Resources/config/solr/managed-schema.xml"
+    default_config_files[2]="${SCRIPT_DIR}/../src/lib/Resources/config/solr/custom-fields-types-solr9.xml"
+else
+    default_config_files[1]="${SCRIPT_DIR}/../src/lib/Resources/config/solr/schema.xml"
+    default_config_files[2]="${SCRIPT_DIR}/../src/lib/Resources/config/solr/custom-fields-types.xml"
+fi
+
+default_config_files[3]="${SCRIPT_DIR}/../src/lib/Resources/config/solr/language-fieldtypes.xml"
 
 default_cores[0]='core0'
 default_cores[1]='core1'
@@ -16,7 +26,6 @@ default_shards=('shard0')
 
 SOLR_PORT=${SOLR_PORT:-8983}
 SOLR_DIR=${SOLR_DIR:-'__solr'}
-SOLR_VERSION=${SOLR_VERSION:-'8.11.2'}
 SOLR_INSTALL_DIR="${SOLR_DIR}/${SOLR_VERSION}"
 SOLR_DEBUG=${SOLR_DEBUG:-false}
 SOLR_HOME=${SOLR_HOME:-'ezcloud'}
@@ -32,7 +41,13 @@ SOLR_CLOUD=${SOLR_CLOUD:-'no'}
 
 INSTALL_DIR="${SOLR_DIR}/${SOLR_VERSION}"
 HOME_DIR="${INSTALL_DIR}/server/${SOLR_HOME}"
-TEMPLATE_DIR="${HOME_DIR}/template"
+
+if [[ "${SOLR_VERSION}" =~ ^9\. ]]; then
+    TEMPLATE_DIR="${HOME_DIR}/template/conf"
+else
+    TEMPLATE_DIR="${HOME_DIR}/template"
+fi
+
 START_SCRIPT="./${INSTALL_DIR}/bin/solr"
 ZOOKEEPER_CLI_SCRIPT="./${INSTALL_DIR}/server/scripts/cloud-scripts/zkcli.sh"
 ZOOKEEPER_HOST=""
@@ -45,6 +60,9 @@ fi
 download() {
     case ${SOLR_VERSION} in
         # PS!!: Append versions and don't remove old ones (except in major versions), used in integration tests from other packages!
+        9.*)
+            url="https://archive.apache.org/dist/solr/solr/${SOLR_VERSION}/solr-${SOLR_VERSION}.tgz"
+            ;;
         7.7.* | 8.* )
             url="https://archive.apache.org/dist/lucene/solr/${SOLR_VERSION}/solr-${SOLR_VERSION}.tgz"
             ;;
@@ -104,8 +122,8 @@ copy_file() {
 create_dir() {
     local dir_name=$1
 
-    if [ ! -d ${dir_name} ] ; then
-        mkdir ${dir_name} || exit_on_error "Couldn't create directory '${dir_name}'"
+    if [ ! -d "${dir_name}" ] ; then
+        mkdir -p "${dir_name}" || exit_on_error "Couldn't create directory '${dir_name}'"
         echo "Created directory '${dir_name}'"
     fi
 }
@@ -137,7 +155,11 @@ solr_run() {
     echo "Running with version ${SOLR_VERSION} in standalone mode"
     echo "Starting solr on port ${SOLR_PORT}..."
 
-    ./${SOLR_INSTALL_DIR}/bin/solr -p ${SOLR_PORT} -s ${SOLR_HOME} -Dsolr.disable.shardsWhitelist=true || exit_on_error "Can't start Solr"
+    if [[ "${SOLR_VERSION}" =~ ^9\. ]]; then
+        ./${SOLR_INSTALL_DIR}/bin/solr start -p ${SOLR_PORT} -s ${SOLR_HOME} || exit_on_error "Can't start Solr"
+    else
+        ./${SOLR_INSTALL_DIR}/bin/solr -p ${SOLR_PORT} -s ${SOLR_HOME} -Dsolr.disable.shardsWhitelist=true || exit_on_error "Can't start Solr"
+    fi
 
     echo "Started"
 
@@ -162,7 +184,11 @@ solr_create_core() {
     core_name=$1
     config_dir=$2
 
-    ./${SOLR_INSTALL_DIR}/bin/solr create_core -p ${SOLR_PORT} -c ${core_name} -d ${config_dir} || exit_on_error "Can't create core"
+    solr_port_flag="-p ${SOLR_PORT}"
+
+    abs_conf_dir="$(pwd)/${config_dir}"
+
+    ./${SOLR_INSTALL_DIR}/bin/solr create_core ${solr_port_flag} -c ${core_name} -d "${abs_conf_dir}" || exit_on_error "Can't create core"
 }
 
 solr_cloud_configure_nodes() {
@@ -284,10 +310,31 @@ solr_cloud_create_collection() {
 download
 
 if [ "$SOLR_CLOUD" = "no" ]; then
+
+    if [[ "${SOLR_VERSION}" =~ ^9\. ]]; then
+        TEMPLATE_CONF="template/conf"
+    else
+        TEMPLATE_CONF="template"
+    fi
+
+    echo "Generating Solr configuration in ${SOLR_INSTALL_DIR}/server/${SOLR_HOME}/${TEMPLATE_CONF}"
+
+    if [[ "${CORES_SETUP:-}" == "dedicated" || "${SOLR_CLOUD:-}" == "no" ]]; then
+        # dedicated mode: point at the local core path
+        ALLOW_URLS="localhost:${SOLR_PORT}/solr"
+    elif [[ "${SOLR_CLOUD:-}" == "yes" && ${#default_nodes[@]} -gt 0 ]]; then
+        # SolrCloud mode: join all nodes host:port
+        ALLOW_URLS=$(IFS=,; echo "${default_nodes[*]}")
+    else
+        # fallback: single node without core-path
+        ALLOW_URLS="localhost:${SOLR_PORT}"
+    fi
+    export ALLOW_URLS
+
     $SCRIPT_DIR/../bin/generate-solr-config.sh \
             --solr-install-dir="${SOLR_INSTALL_DIR}" \
             --solr-version="${SOLR_VERSION}" \
-            --destination-dir="${SOLR_INSTALL_DIR}/server/${SOLR_HOME}/template"
+            --destination-dir="${SOLR_INSTALL_DIR}/server/${SOLR_HOME}/${TEMPLATE_CONF}"
     solr_run
 else
     solr_cloud_configure_nodes

.github/workflows/integration-tests.yaml

@@ -17,6 +17,7 @@ jobs:
                 solr-version:
                     - '7.7.3'
                     - '8.11.2'
+                    - '9.8.1'
                 cores-setup:
                     - 'dedicated'
                     - 'shared'

bin/generate-solr-config.sh

@@ -7,6 +7,7 @@ DESTINATION_DIR=.platform/configsets/solr8/conf
 SOLR_VERSION=8.11.1
 FORCE=false
 SOLR_INSTALL_DIR=""
+ALLOW_URLS_CLI=""
 
 show_help() {
     cat << EOF
@@ -65,6 +66,9 @@ for i in "$@"; do
             SOLR_INSTALL_DIR="${i#*=}"
             SOLR_INSTALL_DIR="${SOLR_INSTALL_DIR/#\~/$HOME}"
             ;;
+        --allow-urls)
+            ALLOW_URLS_CLI="$2"; shift 2
+            ;;
         -h|--help)
             show_help
             exit 0
@@ -76,6 +80,7 @@ for i in "$@"; do
     esac
 done
 
+: "${ALLOW_URLS_CLI:=${ALLOW_URLS:-}}"
 
 if [ `whoami` == "root" ]; then
     echo "Error : Do not run this script as root"
@@ -112,8 +117,30 @@ cp -a ${EZ_BUNDLE_PATH}/src/lib/Resources/config/solr/* $DESTINATION_DIR
 cp ${SOLR_INSTALL_DIR}/server/solr/configsets/_default/conf/{solrconfig.xml,stopwords.txt,synonyms.txt} $DESTINATION_DIR
 
 if [[ ! $DESTINATION_DIR =~ ^\.platform ]]; then
-    # If we are not targeting .platform(.sh) config, we also output default solr.xml
-    cp -f ${SOLR_INSTALL_DIR}/server/solr/solr.xml $DESTINATION_DIR/..
+
+    if [[ "${SOLR_VERSION}" =~ ^9\. ]]; then
+        cp -f ${SOLR_INSTALL_DIR}/server/solr/solr.xml $DESTINATION_DIR/../..
+
+        URL_LIST="${ALLOW_URLS_CLI//,/ }"
+        SOLR_XML_PATH="${DESTINATION_DIR}/../../solr.xml"
+
+        if [[ -f "$SOLR_XML_PATH" ]]; then
+            # backup original
+            cp "$SOLR_XML_PATH" "${SOLR_XML_PATH}.bak"
+            # replace inner text of the allowUrls element
+            sed -i \
+              -e "s|\(<str name=\"allowUrls\">\)[^<]*\(<\/str>\)|\1${URL_LIST}\2|" \
+              "$SOLR_XML_PATH"
+            echo "NOTE: Updated <str name=\"allowUrls\"> to: ${URL_LIST}"
+        else
+            echo "WARNING: solr.xml not found at '$SOLR_XML_PATH'; skipping allowUrls patch"
+        fi
+    else
+        # If we are not targeting .platform(.sh) config, we also output default solr.xml
+        echo "Copying ${SOLR_INSTALL_DIR}/server/solr/solr.xml to $DESTINATION_DIR/.."
+
+        cp -f ${SOLR_INSTALL_DIR}/server/solr/solr.xml $DESTINATION_DIR/..
+    fi
 else
     echo "NOTE: Skipped copying ${SOLR_INSTALL_DIR}/server/solr/solr.xml given destination dir is a '.platform/' config folder"
 fi

src/bundle/DependencyInjection/Configuration.php

@@ -14,6 +14,7 @@ class Configuration implements ConfigurationInterface
 {
     public const SOLR_HTTP_CLIENT_DEFAULT_TIMEOUT = 10;
     public const SOLR_HTTP_CLIENT_DEFAULT_MAX_RETRIES = 3;
+    public const SOLR_DEFAULT_VERSION = '7.7.3';
 
     protected $rootNodeName;
 
@@ -105,6 +106,10 @@ protected function addEndpointsSection(ArrayNodeDefinition $node)
     protected function addConnectionsSection(ArrayNodeDefinition $node)
     {
         $node->children()
+            ->scalarNode('version')
+                ->info('Version of the Solr Search Engine to use')
+                ->defaultValue(self::SOLR_DEFAULT_VERSION)
+            ->end()
             ->scalarNode('default_connection')
                 ->info('Name of the default connection')
             ->end()

src/bundle/DependencyInjection/IbexaSolrExtension.php

@@ -159,6 +159,13 @@ protected function processConnectionConfiguration(ContainerBuilder $container, a
             );
         }
 
+        if (isset($config['version'])) {
+            $container->setParameter(
+                "{$alias}.version",
+                $config['version']
+            );
+        }
+
         foreach ($config['connections'] as $name => $params) {
             $this->configureSearchServices($container, $name, $params);
             $this->configureBoostMap($container, $name, $params);

src/bundle/Resources/config/services.yml

@@ -2,6 +2,7 @@ parameters:
     ibexa.solr.default_connection: ~
     ibexa.solr.http_client.timeout: !php/const \Ibexa\Bundle\Solr\DependencyInjection\Configuration::SOLR_HTTP_CLIENT_DEFAULT_TIMEOUT
     ibexa.solr.http_client.max_retries: !php/const \Ibexa\Bundle\Solr\DependencyInjection\Configuration::SOLR_HTTP_CLIENT_DEFAULT_MAX_RETRIES
+    ibexa.solr.version: !php/const \Ibexa\Bundle\Solr\DependencyInjection\Configuration::SOLR_DEFAULT_VERSION
 
 services:
     ibexa.solr.http_client.retryable:

src/lib/Query/Common/CriterionVisitor/MapLocation/MapLocationDistanceRange.php

@@ -10,13 +10,29 @@
 use Ibexa\Contracts\Core\Repository\Values\Content\Query\Criterion\Operator;
 use Ibexa\Contracts\Solr\Query\CriterionVisitor;
 use Ibexa\Core\Base\Exceptions\InvalidArgumentException;
+use Ibexa\Core\Search\Common\FieldNameResolver;
 use Ibexa\Solr\Query\Common\CriterionVisitor\MapLocation;
 
 /**
  * Visits the MapLocationDistance criterion.
  */
 class MapLocationDistanceRange extends MapLocation
 {
+    private const MAX_EARTH_DISTANCE_KM = 63510;
+
+    private string $solrVersion;
+
+    public function __construct(
+        FieldNameResolver $fieldNameResolver,
+        $fieldTypeIdentifier,
+        $fieldName,
+        string $solrVersion
+    ) {
+        parent::__construct($fieldNameResolver, $fieldTypeIdentifier, $fieldName);
+
+        $this->solrVersion = $solrVersion;
+    }
+
     /**
      * Check if visitor is applicable to current criterion.
      *
@@ -29,10 +45,10 @@ public function canVisit(Criterion $criterion)
         return
             $criterion instanceof Criterion\MapLocationDistance &&
             ($criterion->operator === Operator::LT ||
-              $criterion->operator === Operator::LTE ||
-              $criterion->operator === Operator::GT ||
-              $criterion->operator === Operator::GTE ||
-              $criterion->operator === Operator::BETWEEN);
+                $criterion->operator === Operator::LTE ||
+                $criterion->operator === Operator::GT ||
+                $criterion->operator === Operator::GTE ||
+                $criterion->operator === Operator::BETWEEN);
     }
 
     /**
@@ -47,10 +63,13 @@ public function canVisit(Criterion $criterion)
      */
     public function visit(Criterion $criterion, CriterionVisitor $subVisitor = null)
     {
+        if (!$this->isSolrInMaxVersion('9.3.0')) {
+            return $this->visitForSolr9($criterion);
+        }
         $criterion->value = (array)$criterion->value;
 
         $start = $criterion->value[0];
-        $end = isset($criterion->value[1]) ? $criterion->value[1] : 63510;
+        $end = isset($criterion->value[1]) ? $criterion->value[1] : self::MAX_EARTH_DISTANCE_KM;
 
         if (($criterion->operator === Operator::LT) ||
             ($criterion->operator === Operator::LTE)) {
@@ -88,6 +107,66 @@ public function visit(Criterion $criterion, CriterionVisitor $subVisitor = null)
 
         return '(' . implode(' OR ', $queries) . ')';
     }
-}
 
-class_alias(MapLocationDistanceRange::class, 'EzSystems\EzPlatformSolrSearchEngine\Query\Common\CriterionVisitor\MapLocation\MapLocationDistanceRange');
+    private function visitForSolr9(Criterion $criterion): string
+    {
+        if (is_array($criterion->value)) {
+            $minDistance = $criterion->value[0];
+            $maxDistance = $criterion->value[1] ?? self::MAX_EARTH_DISTANCE_KM;
+        } else {
+            $minDistance = 0;
+            $maxDistance = $criterion->value;
+        }
+
+        $sign = '';
+        if (($criterion->operator === Operator::GT) ||
+            ($criterion->operator === Operator::GTE)) {
+            $sign = '-';
+        }
+
+        $searchFields = $this->getSearchFields(
+            $criterion,
+            $criterion->target,
+            $this->fieldTypeIdentifier,
+            $this->fieldName
+        );
+
+        if (empty($searchFields)) {
+            throw new InvalidArgumentException(
+                '$criterion->target',
+                "No searchable Fields found for the provided Criterion target '{$criterion->target}'."
+            );
+        }
+
+        /** @var \Ibexa\Contracts\Core\Repository\Values\Content\Query\Criterion\Value\MapLocationValue $location */
+        $location = $criterion->valueData;
+
+        $queries = [];
+        foreach ($searchFields as $name => $fieldType) {
+            if ($criterion->operator === Operator::BETWEEN) {
+                $query = sprintf(
+                    '{!geofilt sfield=%s pt=%F,%F d=%s} AND -{!geofilt sfield=%s pt=%F,%F d=%s}',
+                    $name,
+                    $location->latitude,
+                    $location->longitude,
+                    $maxDistance,
+                    $name,
+                    $location->latitude,
+                    $location->longitude,
+                    $minDistance
+                );
+            } else {
+                $query = sprintf('%s{!geofilt sfield=%s pt=%F,%F d=%s}', $sign, $name, $location->latitude, $location->longitude, $maxDistance);
+            }
+
+            $queries[] = "{$query} AND {$name}:[* TO *]";
+        }
+
+        return '(' . implode(' OR ', $queries) . ')';
+    }
+
+    private function isSolrInMaxVersion(string $maxVersion): bool
+    {
+        return version_compare($this->solrVersion, $maxVersion, '<');
+    }
+}

src/lib/Resources/config/container/solr/criterion_visitors.yml

@@ -115,6 +115,7 @@ services:
             - '@Ibexa\Core\Search\Common\FieldNameResolver'
             - 'ezgmaplocation'
             - 'value_location'
+            - '%ibexa.solr.version%'
         tags:
             - {name: ibexa.search.solr.query.content.criterion.visitor}
             - {name: ibexa.search.solr.query.location.criterion.visitor}

src/lib/Resources/config/solr/custom-fields-types-solr9.xml

@@ -0,0 +1,18 @@
+    <!--
+      This is additional custom example fields. You can come up with similar
+      fields on your own, if you require custom indexing / query rules.
+
+      Instead of using the type "text" you might even want to define a custom
+      type. In the custom type you can define your dedicated index and query
+      rules. You can copy any existing fields into the custom field as seen
+      below.
+
+      In this case we copy the full user name and index it as a text field.
+    -->
+    <field name="custom_field" type="text" indexed="true" stored="false" required="false" multiValued="true" />
+    <copyField source="user_first_name_value_s" dest="custom_field" />
+    <copyField source="user_last_name_value_s" dest="custom_field" />
+
+    <field name="custom_geolocation_field" type="location" indexed="true" stored="false" required="false" />
+
+    <copyField source="testtype_maplocation_value_location_gl" dest="custom_geolocation_field" />