Skip to content

Commit 6a560f2

Browse files
lukmziglukmzig
authored
[Improvement] Add functionality to enqueue dependent items (#252)
* add functionality to enqueue dependent items * Apply php-cs-fixer changes * fix: qodana * Apply php-cs-fixer changes * fix: stan * Apply php-cs-fixer changes * fix: add better check for index name on deletion * fix merge conflicts * Apply php-cs-fixer changes * fix STAN * add function tests * Apply php-cs-fixer changes --------- Co-authored-by: lukmzig <[email protected]>
1 parent f997a03 commit 6a560f2

29 files changed

+644
-79
lines changed

‎config/services/search/search-services.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ services:
4040
Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchResultItem\LazyLoading\AssetLazyLoadingHandlerInterface:
4141
class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchResultItem\LazyLoading\AssetLazyLoadingHandler
4242

43+
Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\RequiredByElementListServiceInterface:
44+
class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchService\RequiredByElementListService
45+
4346
Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchResultItem\LazyLoading\DataObjectLazyLoadingHandlerInterface:
4447
class: Pimcore\Bundle\GenericDataIndexBundle\Service\Search\SearchResultItem\LazyLoading\DataObjectLazyLoadingHandler
4548

‎src/Model/SearchIndex/HitData.php‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
<?php
2+
declare(strict_types=1);
3+
4+
/**
5+
* Pimcore
6+
*
7+
* This source file is available under two different licenses:
8+
* - GNU General Public License version 3 (GPLv3)
9+
* - Pimcore Commercial License (PCL)
10+
* Full copyright and license information is available in
11+
* LICENSE.md which is distributed with this source code.
12+
*
13+
* @copyright Copyright (c) Pimcore GmbH (http://www.pimcore.org)
14+
* @license http://www.pimcore.org/license GPLv3 and PCL
15+
*/
16+
17+
namespace Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex;
18+
19+
final readonly class HitData
20+
{
21+
public function __construct(
22+
private string $id,
23+
private string $elementType,
24+
private string $index
25+
) {
26+
}
27+
28+
public function getId(): string
29+
{
30+
return $this->id;
31+
}
32+
33+
public function getElementType(): string
34+
{
35+
return $this->elementType;
36+
}
37+
38+
public function getIndex(): string
39+
{
40+
return $this->index;
41+
}
42+
}

‎src/Repository/IndexQueueRepository.php‎

Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,16 @@
1717
namespace Pimcore\Bundle\GenericDataIndexBundle\Repository;
1818

1919
use Doctrine\DBAL\Connection;
20+
use Doctrine\DBAL\Exception as DBALException;
2021
use Doctrine\DBAL\Query\QueryBuilder as DBALQueryBuilder;
2122
use Doctrine\ORM\EntityManagerInterface;
2223
use Doctrine\ORM\NonUniqueResultException;
2324
use Doctrine\ORM\NoResultException;
2425
use Doctrine\ORM\QueryBuilder;
2526
use Exception;
2627
use Pimcore\Bundle\GenericDataIndexBundle\Entity\IndexQueue;
28+
use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\IndexQueueOperation;
29+
use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\HitData;
2730
use Pimcore\Bundle\GenericDataIndexBundle\Service\TimeServiceInterface;
2831
use Pimcore\Bundle\GenericDataIndexBundle\Traits\LoggerAwareTrait;
2932
use Symfony\Component\Serializer\Exception\ExceptionInterface;
@@ -105,7 +108,7 @@ public function getUnhandledIndexQueueEntries(
105108
/**
106109
* @param IndexQueue[] $entries
107110
*
108-
* @throws \Doctrine\DBAL\Exception
111+
* @throws DBALException
109112
*/
110113
public function deleteQueueEntries(array $entries): void
111114
{
@@ -166,12 +169,12 @@ public function generateSelectQuery(
166169
}
167170

168171
/**
169-
* @throws \Doctrine\DBAL\Exception
172+
* @throws DBALException
170173
*/
171174
public function enqueueBySelectQuery(DBALQueryBuilder $queryBuilder): void
172175
{
173176
$sql = <<<SQL
174-
INSERT INTO
177+
INSERT INTO
175178
%s (elementId, elementType, elementIndexName, operation, operationTime, dispatched)
176179
%s
177180
ON DUPLICATE KEY
@@ -186,7 +189,45 @@ public function enqueueBySelectQuery(DBALQueryBuilder $queryBuilder): void
186189
}
187190

188191
/**
189-
* @throws \Doctrine\DBAL\Exception
192+
* @throws DBALException
193+
*
194+
* @param HitData[] $enqueueItemList
195+
*/
196+
public function enqueueByItemList(array $enqueueItemList, IndexQueueOperation $operation, int $operationTime): void
197+
{
198+
if (empty($enqueueItemList)) {
199+
return;
200+
}
201+
202+
$sql = <<<SQL
203+
INSERT INTO
204+
%s (elementId, elementType, elementIndexName, operation, operationTime, dispatched)
205+
VALUES %s
206+
ON DUPLICATE KEY
207+
UPDATE
208+
operation = VALUES(operation),
209+
operationTime = VALUES(operationTime),
210+
dispatched = VALUES(dispatched)
211+
SQL;
212+
213+
$values = [];
214+
foreach ($enqueueItemList as $item) {
215+
$values[] = sprintf(
216+
'(%s, %s, %s, %s, %s, 0)',
217+
$this->connection->quote($item->getId()),
218+
$this->connection->quote($item->getElementType()),
219+
$this->connection->quote($item->getIndex()),
220+
$this->connection->quote($operation->value),
221+
$operationTime
222+
);
223+
}
224+
$this->connection->executeQuery(
225+
sprintf($sql, IndexQueue::TABLE, implode(',', $values))
226+
);
227+
}
228+
229+
/**
230+
* @throws DBALException
190231
*/
191232
public function dispatchItems(
192233
int $limit

‎src/SearchIndexAdapter/DefaultSearch/DefaultSearchService.php‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
use Pimcore\Bundle\GenericDataIndexBundle\Exception\DefaultSearch\SearchFailedException;
2222
use Pimcore\Bundle\GenericDataIndexBundle\Exception\SwitchIndexAliasException;
2323
use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\Debug\SearchInformation;
24+
use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface;
2425
use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\Search;
2526
use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Interfaces\AdapterSearchInterface;
2627
use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndexAdapter\SearchResult;
@@ -37,9 +38,9 @@
3738
*/
3839
final class DefaultSearchService implements SearchIndexServiceInterface
3940
{
40-
private const INDEX_VERSION_ODD = 'odd';
41+
public const INDEX_VERSION_ODD = 'odd';
4142

42-
private const INDEX_VERSION_EVEN = 'even';
43+
public const INDEX_VERSION_EVEN = 'even';
4344

4445
use LoggerAwareTrait;
4546

@@ -229,7 +230,7 @@ public function createPaginatedSearch(
229230
int $page,
230231
int $pageSize,
231232
bool $aggregationsOnly = false
232-
): AdapterSearchInterface {
233+
): DefaultSearchInterface {
233234
if ($aggregationsOnly) {
234235
return new Search(
235236
from: 0,

‎src/SearchIndexAdapter/DefaultSearch/Search/FetchIdsBySearchService.php‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,14 @@
1616

1717
namespace Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search;
1818

19+
use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\FieldCategory;
1920
use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\FieldCategory\SystemField;
2021
use Pimcore\Bundle\GenericDataIndexBundle\Exception\InvalidArgumentException;
2122
use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface;
2223
use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\Sort\FieldSort;
2324
use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\Sort\FieldSortList;
25+
use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\HitData;
26+
use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndexAdapter\SearchResultHit;
2427
use Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\SearchIndexServiceInterface;
2528
use Pimcore\Bundle\GenericDataIndexBundle\Service\SearchIndex\SearchIndexConfigServiceInterface;
2629

@@ -49,6 +52,26 @@ public function fetchAllIds(DefaultSearchInterface $search, string $indexName, b
4952
return $this->doFetchIds($search, $indexName);
5053
}
5154

55+
/**
56+
* @return HitData[]
57+
*/
58+
public function fetchAllTypesAndIds(
59+
DefaultSearchInterface $search,
60+
string $indexName,
61+
bool $sortById = true
62+
): array {
63+
$search = clone $search;
64+
if ($sortById) {
65+
$search->setSortList(new FieldSortList([new FieldSort(SystemField::ID->getPath())]));
66+
}
67+
68+
if ($search->getSortList()->isEmpty()) {
69+
throw new InvalidArgumentException('Search must have a sort defined to be able to fetch all ids');
70+
}
71+
72+
return $this->doFetchIdsAndTypes($search, $indexName);
73+
}
74+
5275
private function doFetchIds(DefaultSearchInterface $search, string $indexName, ?array $searchAfter = null): array
5376
{
5477
$search->setFrom(0);
@@ -66,6 +89,34 @@ private function doFetchIds(DefaultSearchInterface $search, string $indexName, ?
6689
return $ids;
6790
}
6891

92+
private function doFetchIdsAndTypes(
93+
DefaultSearchInterface $search,
94+
string $indexName,
95+
?array $searchAfter = null
96+
): array {
97+
$search->setFrom(0);
98+
$search->setSize($this->getPageSize());
99+
$search->setSource([SystemField::ELEMENT_TYPE->getPath()]);
100+
$search->setSearchAfter($searchAfter);
101+
$searchResult = $this->searchIndexService->search($search, $indexName);
102+
$hits = $searchResult->getHits();
103+
$idsAndTypes = array_map(
104+
static fn (SearchResultHit $item) =>
105+
new HitData(
106+
id: $item->getId(),
107+
elementType: $item->getSource()[FieldCategory::SYSTEM_FIELDS->value][SystemField::ELEMENT_TYPE->value],
108+
index: $item->getIndex(),
109+
),
110+
$hits
111+
);
112+
$lastHit = $searchResult->getLastHit();
113+
if ($lastHit && (count($hits) === $this->getPageSize())) {
114+
return array_merge($idsAndTypes, $this->doFetchIdsAndTypes($search, $indexName, $lastHit->getSort()));
115+
}
116+
117+
return $idsAndTypes;
118+
}
119+
69120
private function getPageSize(): int
70121
{
71122
$maxResultWindow = $this->searchIndexConfigService->getIndexSettings()['max_result_window'];

‎src/SearchIndexAdapter/DefaultSearch/Search/FetchIdsBySearchServiceInterface.php‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,18 @@
1717
namespace Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter\DefaultSearch\Search;
1818

1919
use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface;
20+
use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndex\HitData;
2021

2122
interface FetchIdsBySearchServiceInterface
2223
{
2324
public function fetchAllIds(DefaultSearchInterface $search, string $indexName, bool $sortById = true): array;
25+
26+
/**
27+
* @return HitData[]
28+
*/
29+
public function fetchAllTypesAndIds(
30+
DefaultSearchInterface $search,
31+
string $indexName,
32+
bool $sortById = true
33+
): array;
2434
}

‎src/SearchIndexAdapter/SearchIndexServiceInterface.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
namespace Pimcore\Bundle\GenericDataIndexBundle\SearchIndexAdapter;
1818

1919
use Exception;
20+
use Pimcore\Bundle\GenericDataIndexBundle\Model\DefaultSearch\DefaultSearchInterface;
2021
use Pimcore\Bundle\GenericDataIndexBundle\Model\Search\Interfaces\AdapterSearchInterface;
2122
use Pimcore\Bundle\GenericDataIndexBundle\Model\SearchIndexAdapter\SearchResult;
2223

@@ -56,7 +57,7 @@ public function createPaginatedSearch(
5657
int $page,
5758
int $pageSize,
5859
bool $aggregationsOnly = false
59-
): AdapterSearchInterface;
60+
): DefaultSearchInterface;
6061

6162
public function search(AdapterSearchInterface $search, string $indexName): SearchResult;
6263

‎src/Service/ElementService.php‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@
2121
use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\ElementType;
2222
use Pimcore\Bundle\GenericDataIndexBundle\Exception\InvalidElementTypeException;
2323
use Pimcore\Model\Asset;
24+
use Pimcore\Model\DataObject;
2425
use Pimcore\Model\DataObject\AbstractObject;
2526
use Pimcore\Model\Document;
27+
use Pimcore\Model\Element\ElementInterface;
2628

2729
/**
2830
* @internal
@@ -47,6 +49,19 @@ public function getElementByType(int $id, string $type): Asset|AbstractObject|Do
4749
};
4850
}
4951

52+
/**
53+
* @throws InvalidElementTypeException
54+
*/
55+
public function getElementType(ElementInterface $element): ElementType
56+
{
57+
return match (true) {
58+
$element instanceof Asset => ElementType::ASSET,
59+
$element instanceof Document => ElementType::DOCUMENT,
60+
$element instanceof DataObject => ElementType::DATA_OBJECT,
61+
default => throw new InvalidElementTypeException('Invalid element type: ' . $element->getType())
62+
};
63+
}
64+
5065
public function classDefinitionExists(string $name): bool
5166
{
5267
try {

‎src/Service/ElementServiceInterface.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,12 @@
1616

1717
namespace Pimcore\Bundle\GenericDataIndexBundle\Service;
1818

19+
use Pimcore\Bundle\GenericDataIndexBundle\Enum\SearchIndex\ElementType;
1920
use Pimcore\Bundle\GenericDataIndexBundle\Exception\InvalidElementTypeException;
2021
use Pimcore\Model\Asset;
2122
use Pimcore\Model\DataObject\AbstractObject;
2223
use Pimcore\Model\Document;
24+
use Pimcore\Model\Element\ElementInterface;
2325

2426
/**
2527
* @internal
@@ -33,5 +35,10 @@ interface ElementServiceInterface
3335
*/
3436
public function getElementByType(int $id, string $type): Asset|AbstractObject|Document|null;
3537

38+
/**
39+
* @throws InvalidElementTypeException
40+
*/
41+
public function getElementType(ElementInterface $element): ElementType;
42+
3643
public function classDefinitionExists(string $name): bool;
3744
}

‎src/Service/Search/SearchService/Asset/AssetSearchService.php‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,15 +52,15 @@ public function search(
5252
AssetSearchInterface $assetSearch,
5353
PermissionTypes $permissionType = PermissionTypes::LIST
5454
): AssetSearchResult {
55-
$assetSearch = $this->searchHelper->addSearchRestrictions(
55+
$search = $this->searchHelper->addSearchRestrictions(
5656
search: $assetSearch,
5757
userPermission: UserPermissionTypes::ASSETS->value,
5858
workspaceType: AssetWorkspace::WORKSPACE_TYPE,
5959
permissionType: $permissionType
6060
);
6161

6262
$searchResult = $this->searchHelper->performSearch(
63-
search: $assetSearch,
63+
search: $search,
6464
indexName: $this->assetTypeAdapter->getAliasIndexName()
6565
);
6666

@@ -75,12 +75,12 @@ public function search(
7575
items: $this->searchHelper->hydrateSearchResultHits(
7676
$searchResult,
7777
$childrenCounts,
78-
$assetSearch->getUser()
78+
$search->getUser()
7979
),
8080
pagination: $this->paginationInfoService->getPaginationInfoFromSearchResult(
8181
searchResult: $searchResult,
82-
page: $assetSearch->getPage(),
83-
pageSize: $assetSearch->getPageSize()
82+
page: $search->getPage(),
83+
pageSize: $search->getPageSize()
8484
),
8585
aggregations: $searchResult->getAggregations(),
8686
);

0 commit comments

Comments
 (0)