Skip to content

Commit 9ae31b5

Browse files
marcelklehrmarcelklehr
authored
Merge pull request #79 from nextcloud/enh/db-schema-change
enh: db schema change
2 parents b252c8a + e5b7b89 commit 9ae31b5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+1262
-609
lines changed

‎appinfo/routes.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
<?php
2+
23
/**
34
* Nextcloud - ContextChat
45
*

‎composer.lock‎

Lines changed: 235 additions & 146 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎lib/AppInfo/Application.php‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
<?php
2+
23
/**
34
* Nextcloud - ContextChat
45
*
@@ -11,6 +12,8 @@
1112

1213
use OCA\ContextChat\Listener\AppDisableListener;
1314
use OCA\ContextChat\Listener\FileListener;
15+
use OCA\ContextChat\Listener\ShareListener;
16+
use OCA\ContextChat\Listener\UserDeletedListener;
1417
use OCA\ContextChat\Service\ProviderConfigService;
1518
use OCA\ContextChat\TaskProcessing\ContextChatProvider;
1619
use OCA\ContextChat\TaskProcessing\ContextChatTaskType;
@@ -27,11 +30,12 @@
2730
use OCP\IConfig;
2831
use OCP\Share\Events\ShareCreatedEvent;
2932
use OCP\Share\Events\ShareDeletedEvent;
33+
use OCP\User\Events\UserDeletedEvent;
3034

3135
class Application extends App implements IBootstrap {
3236

3337
public const APP_ID = 'context_chat';
34-
public const MIN_APP_API_VERSION = '2.0.3';
38+
public const MIN_APP_API_VERSION = '3.0.0';
3539

3640
public const CC_DEFAULT_REQUEST_TIMEOUT = 60 * 50; // 50 mins
3741
// max size per file + max size of the batch of files to be embedded in a single request
@@ -71,12 +75,13 @@ public function __construct(array $urlParams = []) {
7175
public function register(IRegistrationContext $context): void {
7276
$context->registerEventListener(BeforeNodeDeletedEvent::class, FileListener::class);
7377
$context->registerEventListener(NodeCreatedEvent::class, FileListener::class);
74-
$context->registerEventListener(ShareCreatedEvent::class, FileListener::class);
75-
$context->registerEventListener(ShareDeletedEvent::class, FileListener::class);
7678
$context->registerEventListener(CacheEntryInsertedEvent::class, FileListener::class);
7779
$context->registerEventListener(NodeRemovedFromCache::class, FileListener::class);
7880
$context->registerEventListener(NodeWrittenEvent::class, FileListener::class);
7981
$context->registerEventListener(AppDisableEvent::class, AppDisableListener::class);
82+
$context->registerEventListener(UserDeletedEvent::class, UserDeletedListener::class);
83+
$context->registerEventListener(ShareCreatedEvent::class, ShareListener::class);
84+
$context->registerEventListener(ShareDeletedEvent::class, ShareListener::class);
8085
$context->registerTaskProcessingTaskType(ContextChatTaskType::class);
8186
$context->registerTaskProcessingProvider(ContextChatProvider::class);
8287

‎lib/BackgroundJobs/ActionJob.php‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
<?php
2+
3+
/**
4+
* Nextcloud - ContextChat
5+
*
6+
* This file is licensed under the Affero General Public License version 3 or
7+
* later. See the COPYING file.
8+
*
9+
* @author Anupam Kumar <[email protected]>
10+
* @copyright Anupam Kumar 2024
11+
*/
12+
13+
declare(strict_types=1);
14+
namespace OCA\ContextChat\BackgroundJobs;
15+
16+
use OCA\ContextChat\Db\QueueActionMapper;
17+
use OCA\ContextChat\Service\DiagnosticService;
18+
use OCA\ContextChat\Service\LangRopeService;
19+
use OCA\ContextChat\Type\ActionType;
20+
use OCP\AppFramework\Utility\ITimeFactory;
21+
use OCP\BackgroundJob\IJobList;
22+
use OCP\BackgroundJob\QueuedJob;
23+
use Psr\Log\LoggerInterface;
24+
25+
class ActionJob extends QueuedJob {
26+
private const BATCH_SIZE = 100;
27+
28+
public function __construct(
29+
ITimeFactory $timeFactory,
30+
private LangRopeService $networkService,
31+
private QueueActionMapper $actionMapper,
32+
private IJobList $jobList,
33+
private LoggerInterface $logger,
34+
private DiagnosticService $diagnosticService,
35+
) {
36+
parent::__construct($timeFactory);
37+
}
38+
39+
protected function run($argument): void {
40+
$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
41+
$entities = $this->actionMapper->getFromQueue(static::BATCH_SIZE);
42+
43+
if (empty($entities)) {
44+
return;
45+
}
46+
47+
try {
48+
foreach ($entities as $entity) {
49+
$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
50+
51+
switch ($entity->getType()) {
52+
case ActionType::DELETE_SOURCE_IDS:
53+
$decoded = json_decode($entity->getPayload(), true);
54+
if (!is_array($decoded) || !isset($decoded['sourceIds'])) {
55+
$this->logger->warning('Invalid payload for DELETE_SOURCE_IDS action', ['payload' => $entity->getPayload()]);
56+
break;
57+
}
58+
$this->networkService->deleteSources($decoded['sourceIds']);
59+
break;
60+
61+
case ActionType::DELETE_PROVIDER_ID:
62+
$decoded = json_decode($entity->getPayload(), true);
63+
if (!is_array($decoded) || !isset($decoded['providerId'])) {
64+
$this->logger->warning('Invalid payload for DELETE_PROVIDER_ID action', ['payload' => $entity->getPayload()]);
65+
break;
66+
}
67+
$this->networkService->deleteProvider($decoded['providerId']);
68+
break;
69+
70+
case ActionType::DELETE_USER_ID:
71+
$decoded = json_decode($entity->getPayload(), true);
72+
if (!is_array($decoded) || !isset($decoded['userId'])) {
73+
$this->logger->warning('Invalid payload for DELETE_USER_ID action', ['payload' => $entity->getPayload()]);
74+
break;
75+
}
76+
$this->networkService->deleteUser($decoded['userId']);
77+
break;
78+
79+
case ActionType::UPDATE_ACCESS_SOURCE_ID:
80+
$decoded = json_decode($entity->getPayload(), true);
81+
if (!is_array($decoded) || !isset($decoded['op']) || !isset($decoded['userIds']) || !isset($decoded['sourceId'])) {
82+
$this->logger->warning('Invalid payload for UPDATE_ACCESS_SOURCE_ID action', ['payload' => $entity->getPayload()]);
83+
break;
84+
}
85+
$this->networkService->updateAccess($decoded['op'], $decoded['userIds'], $decoded['sourceId']);
86+
break;
87+
88+
case ActionType::UPDATE_ACCESS_PROVIDER_ID:
89+
$decoded = json_decode($entity->getPayload(), true);
90+
if (!is_array($decoded) || !isset($decoded['op']) || !isset($decoded['userIds']) || !isset($decoded['providerId'])) {
91+
$this->logger->warning('Invalid payload for UPDATE_ACCESS_PROVIDER_ID action', ['payload' => $entity->getPayload()]);
92+
break;
93+
}
94+
$this->networkService->updateAccessProvider($decoded['op'], $decoded['userIds'], $decoded['providerId']);
95+
break;
96+
97+
case ActionType::UPDATE_ACCESS_DECL_SOURCE_ID:
98+
$decoded = json_decode($entity->getPayload(), true);
99+
if (!is_array($decoded) || !isset($decoded['userIds']) || !isset($decoded['sourceId'])) {
100+
$this->logger->warning('Invalid payload for UPDATE_ACCESS_DECL_SOURCE_ID action', ['payload' => $entity->getPayload()]);
101+
break;
102+
}
103+
$this->networkService->updateAccessDeclarative($decoded['userIds'], $decoded['sourceId']);
104+
break;
105+
106+
default:
107+
$this->logger->warning('Unknown action type', ['type' => $entity->getType()]);
108+
}
109+
}
110+
111+
foreach ($entities as $entity) {
112+
$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
113+
$this->actionMapper->removeFromQueue($entity);
114+
}
115+
} catch (\Throwable $e) {
116+
$this->jobList->add(static::class);
117+
throw $e;
118+
}
119+
120+
$this->jobList->add(static::class);
121+
}
122+
}

‎lib/BackgroundJobs/DeleteJob.php‎

Lines changed: 0 additions & 74 deletions
This file was deleted.

‎lib/BackgroundJobs/IndexerJob.php‎

Lines changed: 70 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
<?php
2+
23
/*
34
* Copyright (c) 2022 The Recognize contributors.
45
* This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
@@ -7,6 +8,7 @@
78

89
namespace OCA\ContextChat\BackgroundJobs;
910

11+
use OCA\ContextChat\AppInfo\Application;
1012
use OCA\ContextChat\Db\QueueFile;
1113
use OCA\ContextChat\Service\DiagnosticService;
1214
use OCA\ContextChat\Service\LangRopeService;
@@ -46,15 +48,15 @@ class IndexerJob extends TimedJob {
4648
public const DEFAULT_MAX_JOBS_COUNT = 3;
4749

4850
public function __construct(
49-
ITimeFactory $time,
51+
ITimeFactory $time,
5052
private LoggerInterface $logger,
51-
private QueueService $queue,
53+
private QueueService $queue,
5254
private IUserMountCache $userMountCache,
53-
private IJobList $jobList,
55+
private IJobList $jobList,
5456
private LangRopeService $langRopeService,
55-
private StorageService $storageService,
56-
private IRootFolder $rootFolder,
57-
private IAppConfig $appConfig,
57+
private StorageService $storageService,
58+
private IRootFolder $rootFolder,
59+
private IAppConfig $appConfig,
5860
private DiagnosticService $diagnosticService,
5961
private IDBConnection $db,
6062
private ITimeFactory $timeFactory,
@@ -178,49 +180,82 @@ protected function hasEnoughRunningJobs(): bool {
178180
protected function index(array $files): void {
179181
$maxTime = $this->getMaxIndexingTime();
180182
$startTime = time();
183+
$sources = [];
184+
$allSourceIds = [];
185+
$loadedSources = [];
186+
$retryQFiles = [];
187+
$size = 0;
188+
181189
foreach ($files as $queueFile) {
182190
$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
183191
if ($startTime + $maxTime < time()) {
184192
break;
185193
}
194+
186195
$file = current($this->rootFolder->getById($queueFile->getFileId()));
187196
if (!$file instanceof File) {
188197
continue;
189198
}
199+
200+
$file_size = $file->getSize();
201+
if ($size + $file_size > Application::CC_MAX_SIZE || count($sources) >= Application::CC_MAX_FILES) {
202+
$loadedSources = array_merge($loadedSources, $this->langRopeService->indexSources($sources));
203+
$sources = [];
204+
$size = 0;
205+
}
206+
190207
$userIds = $this->storageService->getUsersForFileId($queueFile->getFileId());
191-
foreach ($userIds as $userId) {
192-
$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
208+
$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
209+
210+
try {
193211
try {
194-
try {
195-
$fileHandle = $file->fopen('r');
196-
} catch (LockedException|NotPermittedException $e) {
197-
$this->logger->error('Could not open file ' . $file->getPath() . ' for reading', ['exception' => $e]);
198-
continue;
199-
}
200-
if (!is_resource($fileHandle)) {
201-
$this->logger->warning('File handle for' . $file->getPath() . ' is not readable');
202-
continue;
203-
}
204-
$source = new Source(
205-
$userId,
206-
ProviderConfigService::getSourceId($file->getId()),
207-
$file->getPath(),
208-
$fileHandle,
209-
$file->getMtime(),
210-
$file->getMimeType(),
211-
ProviderConfigService::getDefaultProviderKey(),
212-
);
213-
} catch (InvalidPathException|NotFoundException $e) {
214-
$this->logger->error('Could not find file ' . $file->getPath(), ['exception' => $e]);
215-
continue 2;
212+
$fileHandle = $file->fopen('r');
213+
} catch (NotPermittedException $e) {
214+
$this->logger->error('Could not open file ' . $file->getPath() . ' for reading', ['exception' => $e]);
215+
continue;
216+
} catch (LockedException $e) {
217+
$retryQFiles[] = $queueFile;
218+
$this->logger->info('File ' . $file->getPath() . ' is locked, could not read for indexing. Adding it to the next batch.');
219+
continue;
216220
}
217-
$this->langRopeService->indexSources([$source]);
221+
if (!is_resource($fileHandle)) {
222+
$this->logger->warning('File handle for' . $file->getPath() . ' is not readable');
223+
continue;
224+
}
225+
226+
$sources[] = new Source(
227+
$userIds,
228+
ProviderConfigService::getSourceId($file->getId()),
229+
substr($file->getInternalPath(), 6), // remove 'files/' prefix
230+
$fileHandle,
231+
$file->getMtime(),
232+
$file->getMimeType(),
233+
ProviderConfigService::getDefaultProviderKey(),
234+
);
235+
$allSourceIds[] = ProviderConfigService::getSourceId($file->getId());
236+
} catch (InvalidPathException|NotFoundException $e) {
237+
$this->logger->error('Could not find file ' . $file->getPath(), ['exception' => $e]);
238+
continue;
218239
}
219-
try {
220-
$this->queue->removeFromQueue($queueFile);
221-
} catch (Exception $e) {
222-
$this->logger->error('Could not remove file from queue', ['exception' => $e]);
240+
}
241+
242+
if (count($sources) > 0) {
243+
$loadedSources = array_merge($loadedSources, $this->langRopeService->indexSources($sources));
244+
}
245+
246+
$emptyInvalidSources = array_diff($allSourceIds, $loadedSources);
247+
if (count($emptyInvalidSources) > 0) {
248+
$this->logger->info('Invalid or empty sources that were not indexed', ['sourceIds' => $emptyInvalidSources]);
249+
}
250+
251+
try {
252+
$this->queue->removeFromQueue($files);
253+
// add files that were locked to the end of the queue
254+
foreach ($retryQFiles as $queueFile) {
255+
$this->queue->insertIntoQueue($queueFile);
223256
}
257+
} catch (Exception $e) {
258+
$this->logger->error('Could not remove indexed files from queue', ['exception' => $e]);
224259
}
225260
}
226261
}

0 commit comments

Comments
 (0)