Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
search_api_opensearch
Manage
Activity
Members
Labels
Plan
Wiki
Custom issue tracker
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Model registry
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Insights
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Terms and privacy
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
project
search_api_opensearch
Commits
1e35a267
Commit
1e35a267
authored
8 months ago
by
Kim Pepper
Browse files
Options
Downloads
Patches
Plain Diff
Issue
#3285438
by achap, varshith, bkildow, kim.pepper, longwave, larowlan:...
parent
dab67587
No related branches found
No related tags found
2 merge requests
!68
[#3339329] Option to disable SSL verification
,
!57
Issue #3285438 by achap, varshith, bkildow, kim.pepper, longwave, larowlan:...
Pipeline
#201327
passed
8 months ago
Stage: build
Stage: validate
Stage: test
Changes
3
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
src/SearchAPI/BackendClient.php
+72
-1
72 additions, 1 deletion
src/SearchAPI/BackendClient.php
tests/src/Kernel/OpenSearchBackendTest.php
+131
-0
131 additions, 0 deletions
tests/src/Kernel/OpenSearchBackendTest.php
tests/src/Kernel/SpellCheckTest.php
+1
-0
1 addition, 0 deletions
tests/src/Kernel/SpellCheckTest.php
with
204 additions
and
1 deletion
src/SearchAPI/BackendClient.php
+
72
−
1
View file @
1e35a267
...
...
@@ -222,9 +222,15 @@ class BackendClient implements BackendClientInterface {
*/
public
function
updateIndex
(
IndexInterface
$index
):
void
{
if
(
$this
->
indexExists
(
$index
))
{
$index
->
clear
();
$result
=
$this
->
indexNeedsClearing
(
$index
);
if
(
$result
)
{
$index
->
clear
();
}
$this
->
updateSettings
(
$index
);
$this
->
updateFieldMapping
(
$index
);
if
(
!
$result
)
{
$index
->
reindex
();
}
}
else
{
$this
->
addIndex
(
$index
);
...
...
@@ -347,4 +353,69 @@ class BackendClient implements BackendClientInterface {
}
}
/**
* Determine whether the index needs clearing.
*
* OpenSearch does not allow changing existing field
* mappings with data but does allow adding new fields.
* OpenSearch has dynamic mappings which means it will automatically
* add types to fields based on the indexed data if no mapping
* is explicitly provided. It will also override the mapping
* if it is wrong, e.g. if you provide a string type but
* the indexed data for the field is a float it will override the
* mapping. In order to make sure the index is not cleared for minor
* changes It's important to make sure the types for fields are correct
* and that any custom fields are explicitly mapped.
*/
private
function
indexNeedsClearing
(
IndexInterface
$index
):
bool
{
$openSearchMapping
=
$this
->
client
->
indices
()
->
getMapping
([
'index'
=>
$this
->
getIndexId
(
$index
),
]);
$drupalMapping
=
$this
->
fieldParamsBuilder
->
mapFieldParams
(
$this
->
getIndexId
(
$index
),
$index
);
// If mappings have yet to be set no need to clear.
if
(
!
isset
(
$openSearchMapping
[
$this
->
getIndexId
(
$index
)][
'mappings'
][
'properties'
]))
{
return
FALSE
;
}
// Recursively check for differences in the mappings between the
// $openSearchMapping and $drupalMapping arrays. Before comparing,
// convert $drupalMapping to a json format returned as an
// associative array, so it is in the same format as the openSearchMapping.
// For example the putMappings method on the OS client expects an empty
// object but getMapping returns an empty array. Ensure float values are
// preserved as json_encode doesn't preserve them by default.
return
$this
->
mappingsHaveDifferences
(
$openSearchMapping
[
$this
->
getIndexId
(
$index
)][
'mappings'
][
'properties'
],
json_decode
(
json_encode
(
$drupalMapping
[
'body'
][
'properties'
],
JSON_PRESERVE_ZERO_FRACTION
),
TRUE
)
);
}
/**
* Recursively diff an associative array to find differences.
*
* If any difference is found bail early.
*/
private
function
mappingsHaveDifferences
(
array
$array1
,
array
$array2
):
bool
{
foreach
(
$array1
as
$key
=>
$value
)
{
if
(
is_array
(
$value
))
{
if
(
!
isset
(
$array2
[
$key
])
||
!
is_array
(
$array2
[
$key
]))
{
return
TRUE
;
}
else
{
$newDiff
=
$this
->
mappingsHaveDifferences
(
$value
,
$array2
[
$key
]);
if
(
!
empty
(
$newDiff
))
{
return
TRUE
;
}
}
}
elseif
(
!
array_key_exists
(
$key
,
$array2
)
||
$array2
[
$key
]
!==
$value
)
{
return
TRUE
;
}
}
return
FALSE
;
}
}
This diff is collapsed.
Click to expand it.
tests/src/Kernel/OpenSearchBackendTest.php
+
131
−
0
View file @
1e35a267
...
...
@@ -4,12 +4,15 @@ declare(strict_types=1);
namespace
Drupal\Tests\search_api_opensearch\Kernel
;
use
Drupal\field\Entity\FieldConfig
;
use
Drupal\field\Entity\FieldStorageConfig
;
use
Drupal\search_api
\Entity\Index
;
use
Drupal\search_api
\Entity\Server
;
use
Drupal\search_api
\Query\QueryInterface
;
use
Drupal\search_api_opensearch_test
\EventSubscriber\OpenSearchEventSubscribers
;
use
Drupal\Tests\search_api
\Functional\ExampleContentTrait
;
use
Drupal\Tests\search_api
\Kernel\BackendTestBase
;
use
OpenSearch\Client
;
/**
* Tests the end-to-end functionality of the backend.
...
...
@@ -39,6 +42,11 @@ class OpenSearchBackendTest extends BackendTestBase {
*/
protected
$indexId
=
'test_opensearch_index'
;
/**
* The index prefix.
*/
protected
string
$prefix
=
'test_'
;
/**
* {@inheritdoc}
*/
...
...
@@ -68,6 +76,129 @@ class OpenSearchBackendTest extends BackendTestBase {
$this
->
searchSuccess
();
}
/**
* Tests making changes to fields on the index.
*
* Adding a new field should not clear the index
* but changing an existing field mapping should.
*/
public
function
testUpdatingIndex
():
void
{
$server
=
Server
::
load
(
$this
->
serverId
);
/** @var \Drupal\search_api_opensearch\Plugin\search_api\backend\OpenSearchBackend $backend */
$backend
=
$server
->
getBackend
();
$osClient
=
$backend
->
getClient
();
$this
->
recreateIndex
();
$this
->
insertExampleContent
();
$this
->
indexItems
(
$this
->
indexId
);
$this
->
refreshIndices
();
// Check that all our items are indexed both
// by tracker and on server.
$index
=
Index
::
load
(
$this
->
indexId
);
$tracker
=
$index
->
getTrackerInstance
();
$this
->
assertEquals
(
5
,
$tracker
->
getIndexedItemsCount
());
$result
=
$this
->
queryAllResults
(
$osClient
);
$this
->
assertEquals
(
5
,
$result
[
'hits'
][
'total'
][
'value'
]);
// Add a new field to the index.
FieldStorageConfig
::
create
([
'field_name'
=>
'new_field'
,
'entity_type'
=>
'entity_test_mulrev_changed'
,
'type'
=>
'string'
,
])
->
save
();
FieldConfig
::
create
([
'entity_type'
=>
'entity_test_mulrev_changed'
,
'field_name'
=>
'new_field'
,
'bundle'
=>
'item'
,
])
->
save
();
/** @var \Drupal\search_api\Utility\FieldsHelperInterface $fieldsHelper */
$fieldsHelper
=
\Drupal
::
getContainer
()
->
get
(
'search_api.fields_helper'
);
$field
=
$fieldsHelper
->
createField
(
$index
,
'new_field'
,
[
'label'
=>
'New field'
,
'type'
=>
'string'
,
'property_path'
=>
'new_field'
,
'datasource_id'
=>
'entity:entity_test_mulrev_changed'
,
]);
$index
->
addField
(
$field
);
$index
->
save
();
// Now check that the tracker is reset but items remain on
// server.
$this
->
assertEquals
(
0
,
$tracker
->
getIndexedItemsCount
());
$result
=
$this
->
queryAllResults
(
$osClient
);
$this
->
assertEquals
(
5
,
$result
[
'hits'
][
'total'
][
'value'
]);
// Re-index the items for next test.
$this
->
indexItems
(
$this
->
indexId
);
$this
->
refreshIndices
();
$this
->
assertEquals
(
5
,
$tracker
->
getIndexedItemsCount
());
$result
=
$this
->
queryAllResults
(
$osClient
);
$this
->
assertEquals
(
5
,
$result
[
'hits'
][
'total'
][
'value'
]);
// Changing the fields type should clear the index completely
// as OS doesn't allow changes to existing fields with data.
$field
=
$index
->
getField
(
'new_field'
);
$field
->
setType
(
'text'
);
$index
->
addField
(
$field
);
$index
->
save
();
$this
->
assertEquals
(
0
,
$tracker
->
getIndexedItemsCount
());
$result
=
$this
->
queryAllResults
(
$osClient
);
$this
->
assertEquals
(
0
,
$result
[
'hits'
][
'total'
][
'value'
]);
// Re-index the items for next test.
$this
->
indexItems
(
$this
->
indexId
);
$this
->
refreshIndices
();
$this
->
assertEquals
(
5
,
$tracker
->
getIndexedItemsCount
());
$result
=
$this
->
queryAllResults
(
$osClient
);
$this
->
assertEquals
(
5
,
$result
[
'hits'
][
'total'
][
'value'
]);
// Add another new field and change the first field
// back to its original type.
FieldStorageConfig
::
create
([
'field_name'
=>
'new_field_2'
,
'entity_type'
=>
'entity_test_mulrev_changed'
,
'type'
=>
'string'
,
])
->
save
();
FieldConfig
::
create
([
'entity_type'
=>
'entity_test_mulrev_changed'
,
'field_name'
=>
'new_field_2'
,
'bundle'
=>
'item'
,
])
->
save
();
$fieldsHelper
=
\Drupal
::
getContainer
()
->
get
(
'search_api.fields_helper'
);
$field
=
$fieldsHelper
->
createField
(
$index
,
'new_field_2'
,
[
'label'
=>
'New field 2'
,
'type'
=>
'string'
,
'property_path'
=>
'new_field_2'
,
'datasource_id'
=>
'entity:entity_test_mulrev_changed'
,
]);
$index
->
addField
(
$field
);
$existingField
=
$index
->
getField
(
'new_field'
);
$existingField
->
setType
(
'string'
);
$index
->
addField
(
$existingField
);
$index
->
save
();
// The index should be completely cleared because we have both
// a new field and a changed field.
$this
->
assertEquals
(
0
,
$tracker
->
getIndexedItemsCount
());
$result
=
$this
->
queryAllResults
(
$osClient
);
$this
->
assertEquals
(
0
,
$result
[
'hits'
][
'total'
][
'value'
]);
}
/**
* Returns the OS response for all documents.
*/
protected
function
queryAllResults
(
Client
$client
):
array
{
return
$client
->
search
([
'index'
=>
$this
->
prefix
.
$this
->
indexId
,
'body'
=>
[
'query'
=>
[
'match_all'
=>
new
\StdClass
(),
],
],
'track_total_hits'
=>
TRUE
,
]);
}
/**
* Re-creates the index.
*
...
...
This diff is collapsed.
Click to expand it.
tests/src/Kernel/SpellCheckTest.php
+
1
−
0
View file @
1e35a267
...
...
@@ -44,6 +44,7 @@ class SpellCheckTest extends KernelTestBase {
public
function
setUp
():
void
{
parent
::
setUp
();
$this
->
installSchema
(
'search_api'
,
[
'search_api_item'
]);
$this
->
installEntitySchema
(
'entity_test_mulrev_changed'
);
$this
->
installEntitySchema
(
'search_api_task'
);
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment