Skip to content
Snippets Groups Projects
Commit e9c0b841 authored by Andrew Chappell's avatar Andrew Chappell Committed by Kim Pepper
Browse files

Issue #3285438 by achap, varshith, bkildow, kim.pepper, longwave, larowlan:...

Issue #3285438 by achap, varshith, bkildow, kim.pepper, longwave, larowlan: The whole index gets cleared/deleted when any change in the search index configuration is imported/synced
parent 705fa440
No related branches found
No related tags found
2 merge requests!75Allow for rank queries if type boose is active,!52Resolve #3285438 "The whole index"
Pipeline #200593 failed
......@@ -222,9 +222,15 @@ class BackendClient implements BackendClientInterface {
*/
public function updateIndex(IndexInterface $index): void {
if ($this->indexExists($index)) {
$index->clear();
$result = $this->indexNeedsClearing($index);
if ($result) {
$index->clear();
}
$this->updateSettings($index);
$this->updateFieldMapping($index);
if (!$result) {
$index->reindex();
}
}
else {
$this->addIndex($index);
......@@ -347,4 +353,69 @@ class BackendClient implements BackendClientInterface {
}
}
/**
* Determine whether the index needs clearing.
*
* OpenSearch does not allow changing existing field
* mappings with data but does allow adding new fields.
* OpenSearch has dynamic mappings which means it will automatically
* add types to fields based on the indexed data if no mapping
* is explicitly provided. It will also override the mapping
* if it is wrong, e.g. if you provide a string type but
* the indexed data for the field is a float it will override the
* mapping. In order to make sure the index is not cleared for minor
* changes It's important to make sure the types for fields are correct
* and that any custom fields are explicitly mapped.
*/
private function indexNeedsClearing(IndexInterface $index): bool {
$openSearchMapping = $this->client->indices()->getMapping([
'index' => $this->getIndexId($index),
]);
$drupalMapping = $this->fieldParamsBuilder->mapFieldParams($this->getIndexId($index), $index);
// If mappings have yet to be set no need to clear.
if (!isset($openSearchMapping[$this->getIndexId($index)]['mappings']['properties'])) {
return FALSE;
}
// Recursively check for differences in the mappings between the
// $openSearchMapping and $drupalMapping arrays. Before comparing,
// convert $drupalMapping to a json format returned as an
// associative array, so it is in the same format as the openSearchMapping.
// For example the putMappings method on the OS client expects an empty
// object but getMapping returns an empty array. Ensure float values are
// preserved as json_encode doesn't preserve them by default.
return $this->mappingsHaveDifferences(
$openSearchMapping[$this->getIndexId($index)]['mappings']['properties'],
json_decode(
json_encode($drupalMapping['body']['properties'], JSON_PRESERVE_ZERO_FRACTION),
TRUE
)
);
}
/**
* Recursively diff an associative array to find differences.
*
* If any difference is found bail early.
*/
private function mappingsHaveDifferences(array $array1, array $array2): bool {
foreach ($array1 as $key => $value) {
if (is_array($value)) {
if (!isset($array2[$key]) || !is_array($array2[$key])) {
return TRUE;
}
else {
$newDiff = $this->mappingsHaveDifferences($value, $array2[$key]);
if (!empty($newDiff)) {
return TRUE;
}
}
}
elseif (!array_key_exists($key, $array2) || $array2[$key] !== $value) {
return TRUE;
}
}
return FALSE;
}
}
......@@ -4,12 +4,15 @@ declare(strict_types=1);
namespace Drupal\Tests\search_api_opensearch\Kernel;
use Drupal\field\Entity\FieldConfig;
use Drupal\field\Entity\FieldStorageConfig;
use Drupal\search_api\Entity\Index;
use Drupal\search_api\Entity\Server;
use Drupal\search_api\Query\QueryInterface;
use Drupal\search_api_opensearch_test\EventSubscriber\OpenSearchEventSubscribers;
use Drupal\Tests\search_api\Functional\ExampleContentTrait;
use Drupal\Tests\search_api\Kernel\BackendTestBase;
use OpenSearch\Client;
/**
* Tests the end-to-end functionality of the backend.
......@@ -39,6 +42,11 @@ class OpenSearchBackendTest extends BackendTestBase {
*/
protected $indexId = 'test_opensearch_index';
/**
* The index prefix.
*/
protected string $prefix = 'test_';
/**
* {@inheritdoc}
*/
......@@ -68,6 +76,129 @@ class OpenSearchBackendTest extends BackendTestBase {
$this->searchSuccess();
}
/**
* Tests making changes to fields on the index.
*
* Adding a new field should not clear the index
* but changing an existing field mapping should.
*/
public function testUpdatingIndex(): void {
$server = Server::load($this->serverId);
/** @var \Drupal\search_api_opensearch\Plugin\search_api\backend\OpenSearchBackend $backend */
$backend = $server->getBackend();
$osClient = $backend->getClient();
$this->recreateIndex();
$this->insertExampleContent();
$this->indexItems($this->indexId);
$this->refreshIndices();
// Check that all our items are indexed both
// by tracker and on server.
$index = Index::load($this->indexId);
$tracker = $index->getTrackerInstance();
$this->assertEquals(5, $tracker->getIndexedItemsCount());
$result = $this->queryAllResults($osClient);
$this->assertEquals(5, $result['hits']['total']['value']);
// Add a new field to the index.
FieldStorageConfig::create([
'field_name' => 'new_field',
'entity_type' => 'entity_test_mulrev_changed',
'type' => 'string',
])->save();
FieldConfig::create([
'entity_type' => 'entity_test_mulrev_changed',
'field_name' => 'new_field',
'bundle' => 'item',
])->save();
/** @var \Drupal\search_api\Utility\FieldsHelperInterface $fieldsHelper */
$fieldsHelper = \Drupal::getContainer()->get('search_api.fields_helper');
$field = $fieldsHelper->createField($index, 'new_field', [
'label' => 'New field',
'type' => 'string',
'property_path' => 'new_field',
'datasource_id' => 'entity:entity_test_mulrev_changed',
]);
$index->addField($field);
$index->save();
// Now check that the tracker is reset but items remain on
// server.
$this->assertEquals(0, $tracker->getIndexedItemsCount());
$result = $this->queryAllResults($osClient);
$this->assertEquals(5, $result['hits']['total']['value']);
// Re-index the items for next test.
$this->indexItems($this->indexId);
$this->refreshIndices();
$this->assertEquals(5, $tracker->getIndexedItemsCount());
$result = $this->queryAllResults($osClient);
$this->assertEquals(5, $result['hits']['total']['value']);
// Changing the fields type should clear the index completely
// as OS doesn't allow changes to existing fields with data.
$field = $index->getField('new_field');
$field->setType('text');
$index->addField($field);
$index->save();
$this->assertEquals(0, $tracker->getIndexedItemsCount());
$result = $this->queryAllResults($osClient);
$this->assertEquals(0, $result['hits']['total']['value']);
// Re-index the items for next test.
$this->indexItems($this->indexId);
$this->refreshIndices();
$this->assertEquals(5, $tracker->getIndexedItemsCount());
$result = $this->queryAllResults($osClient);
$this->assertEquals(5, $result['hits']['total']['value']);
// Add another new field and change the first field
// back to its original type.
FieldStorageConfig::create([
'field_name' => 'new_field_2',
'entity_type' => 'entity_test_mulrev_changed',
'type' => 'string',
])->save();
FieldConfig::create([
'entity_type' => 'entity_test_mulrev_changed',
'field_name' => 'new_field_2',
'bundle' => 'item',
])->save();
$fieldsHelper = \Drupal::getContainer()->get('search_api.fields_helper');
$field = $fieldsHelper->createField($index, 'new_field_2', [
'label' => 'New field 2',
'type' => 'string',
'property_path' => 'new_field_2',
'datasource_id' => 'entity:entity_test_mulrev_changed',
]);
$index->addField($field);
$existingField = $index->getField('new_field');
$existingField->setType('string');
$index->addField($existingField);
$index->save();
// The index should be completely cleared because we have both
// a new field and a changed field.
$this->assertEquals(0, $tracker->getIndexedItemsCount());
$result = $this->queryAllResults($osClient);
$this->assertEquals(0, $result['hits']['total']['value']);
}
/**
* Returns the OS response for all documents.
*/
protected function queryAllResults(Client $client): array {
return $client->search([
'index' => $this->prefix . $this->indexId,
'body' => [
'query' => [
'match_all' => new \StdClass(),
],
],
'track_total_hits' => TRUE,
]);
}
/**
* Re-creates the index.
*
......
......@@ -44,6 +44,7 @@ class SpellCheckTest extends KernelTestBase {
public function setUp(): void {
parent::setUp();
$this->installSchema('search_api', ['search_api_item']);
$this->installEntitySchema('entity_test_mulrev_changed');
$this->installEntitySchema('search_api_task');
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment