Skip to content

Commit 0a0631c

Browse files
kyteinskykyteinsky
committed
add context_chat:stats for indexing completion time
Signed-off-by: Anupam Kumar <kyteinsky@gmail.com>
1 parent 8ae6863 commit 0a0631c

File tree

10 files changed

+183
-16
lines changed

10 files changed

+183
-16
lines changed

‎appinfo/info.xml‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,14 @@ Refer to the [Context Chat Backend's readme](https://github.com/nextcloud/contex
4040
<job>OCA\ContextChat\BackgroundJobs\SchedulerJob</job>
4141
</background-jobs>
4242
<commands>
43-
<command>OCA\ContextChat\Command\ScanFiles</command>
44-
<command>OCA\ContextChat\Command\Prompt</command>
4543
<command>OCA\ContextChat\Command\Diagnostics</command>
44+
<command>OCA\ContextChat\Command\Prompt</command>
45+
<command>OCA\ContextChat\Command\ScanFiles</command>
46+
<command>OCA\ContextChat\Command\Statistics</command>
4647
</commands>
47-
<!--settings>
48-
<admin>OCA\ContextChat\Settings\Admin</admin>
49-
<admin-section>OCA\ContextChat\Settings\AdminSection</admin-section>
50-
<personal>OCA\ContextChat\Settings\Personal</personal>
51-
<personal-section>OCA\ContextChat\Settings\PersonalSection</personal-section>
52-
</settings-->
48+
<repair-steps>
49+
<install>
50+
<step>OCA\ContextChat\Repair\AppInstallStep</step>
51+
</install>
52+
</repair-steps>
5353
</info>

‎lib/AppInfo/Application.php‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
use OCA\ContextChat\Listener\FileListener;
1515
use OCA\ContextChat\Listener\ShareListener;
1616
use OCA\ContextChat\Listener\UserDeletedListener;
17-
use OCA\ContextChat\Service\ProviderConfigService;
1817
use OCA\ContextChat\TaskProcessing\ContextChatProvider;
1918
use OCA\ContextChat\TaskProcessing\ContextChatTaskType;
2019
use OCP\App\Events\AppDisableEvent;
@@ -84,10 +83,6 @@ public function register(IRegistrationContext $context): void {
8483
$context->registerEventListener(ShareDeletedEvent::class, ShareListener::class);
8584
$context->registerTaskProcessingTaskType(ContextChatTaskType::class);
8685
$context->registerTaskProcessingProvider(ContextChatProvider::class);
87-
88-
$providerConfigService = new ProviderConfigService($this->config);
89-
/** @psalm-suppress ArgumentTypeCoercion, UndefinedClass */
90-
$providerConfigService->updateProvider('files', 'default', '', true);
9186
}
9287

9388
public function boot(IBootContext $context): void {

‎lib/BackgroundJobs/IndexerJob.php‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ public function run($argument): void {
8282
if ($this->appConfig->getAppValue('auto_indexing', 'true') === 'false') {
8383
return;
8484
}
85+
$this->setInitialIndexCompletion();
8586
if ($this->hasEnoughRunningJobs()) {
8687
return;
8788
}
@@ -258,4 +259,28 @@ protected function index(array $files): void {
258259
$this->logger->error('Could not remove indexed files from queue', ['exception' => $e]);
259260
}
260261
}
262+
263+
private function setInitialIndexCompletion(): void {
264+
// if last indexed time is already set, we don't need to do anything
265+
if ($this->appConfig->getAppValueInt('last_indexed_time', 0, false) !== 0) {
266+
return;
267+
}
268+
269+
// if any storage crawler jobs are running, we don't need to do anything
270+
if ($this->jobList->hasReservedJob(StorageCrawlJob::class)) {
271+
return;
272+
}
273+
274+
// if the last indexed file id is in the queue, we don't need to do anything
275+
$lastIndexedFileId = $this->appConfig->getAppValueInt('last_indexed_file_id', 0, false);
276+
if ($lastIndexedFileId === 0) {
277+
return;
278+
}
279+
if ($this->queue->existsQueueFileId($lastIndexedFileId)) {
280+
return;
281+
}
282+
283+
$this->logger->info('Initial index completion detected, setting last indexed time');
284+
$this->appConfig->setAppValueInt('last_indexed_time', $this->timeFactory->getTime(), false);
285+
}
261286
}

‎lib/BackgroundJobs/StorageCrawlJob.php‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
declare(strict_types=1);
99
namespace OCA\ContextChat\BackgroundJobs;
1010

11+
use OCA\ContextChat\AppInfo\Application;
1112
use OCA\ContextChat\Db\QueueFile;
1213
use OCA\ContextChat\Service\DiagnosticService;
1314
use OCA\ContextChat\Service\QueueService;
@@ -16,6 +17,7 @@
1617
use OCP\BackgroundJob\IJobList;
1718
use OCP\BackgroundJob\QueuedJob;
1819
use OCP\DB\Exception;
20+
use OCP\IAppConfig;
1921
use Psr\Log\LoggerInterface;
2022

2123
class StorageCrawlJob extends QueuedJob {
@@ -28,6 +30,7 @@ public function __construct(
2830
private IJobList $jobList,
2931
private StorageService $storageService,
3032
private DiagnosticService $diagnosticService,
33+
private IAppConfig $appConfig,
3134
) {
3235
parent::__construct($timeFactory);
3336
}
@@ -71,6 +74,9 @@ protected function run($argument): void {
7174
'override_root' => $overrideRoot,
7275
'last_file_id' => $queueFile->getFileId(),
7376
]);
77+
78+
// the last job to set this value will win
79+
$this->appConfig->setValueInt(Application::APP_ID, 'last_indexed_file_id', $queueFile->getFileId());
7480
}
7581
}
7682
}

‎lib/Command/Statistics.php‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
<?php
2+
3+
/**
4+
* Nextcloud - ContextChat
5+
*
6+
* This file is licensed under the Affero General Public License version 3 or
7+
* later. See the COPYING file.
8+
*
9+
* @author Anupam Kumar <kyteinsky@gmail.com>
10+
* @copyright Anupam Kumar 2024
11+
*/
12+
13+
declare(strict_types=1);
14+
namespace OCA\ContextChat\Command;
15+
16+
use OCA\ContextChat\Service\ActionService;
17+
use OCA\ContextChat\Service\QueueService;
18+
use OCP\AppFramework\Services\IAppConfig;
19+
use Symfony\Component\Console\Command\Command;
20+
use Symfony\Component\Console\Input\InputInterface;
21+
use Symfony\Component\Console\Output\OutputInterface;
22+
23+
class Statistics extends Command {
24+
25+
public function __construct(
26+
private IAppConfig $appConfig,
27+
private ActionService $actionService,
28+
private QueueService $queueService,
29+
) {
30+
parent::__construct();
31+
}
32+
33+
protected function configure() {
34+
$this->setName('context_chat:stats')
35+
->setDescription('Check ContextChat statistics');
36+
}
37+
38+
protected function execute(InputInterface $input, OutputInterface $output) {
39+
$output->writeln('ContextChat statistics:');
40+
if ($this->appConfig->getAppValueInt('last_indexed_time', 0) === 0) {
41+
$output->writeln('The indexing is not complete yet.');
42+
} else {
43+
$installedTime = $this->appConfig->getAppValueInt('installed_time', 0);
44+
$lastIndexedTime = $this->appConfig->getAppValueInt('last_indexed_time', 0);
45+
$indexTime = $lastIndexedTime - $installedTime;
46+
47+
$output->writeln('Installed time: ' . (new \DateTime('@' . $installedTime))->format('Y-m-d H:i') . ' UTC');
48+
$output->writeln('Index complete time: ' . (new \DateTime('@' . $lastIndexedTime))->format('Y-m-d H:i') . ' UTC');
49+
$output->writeln('Total time taken for a complete index: ' . gmdate('H:i', $indexTime) . ' (hh:mm)');
50+
}
51+
52+
$queueCount = $this->queueService->count();
53+
$output->writeln('Files in indexing queue: ' . $queueCount);
54+
55+
$actionsCount = $this->actionService->count();
56+
$output->writeln('Actions in queue: ' . $actionsCount);
57+
return 0;
58+
}
59+
}

‎lib/Db/QueueActionMapper.php‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,4 +70,18 @@ public function insertIntoQueue(QueueAction $item): void {
7070
])
7171
->executeStatement();
7272
}
73+
74+
/**
75+
* @throws \OCP\DB\Exception
76+
*/
77+
public function count() : int {
78+
$qb = $this->db->getQueryBuilder();
79+
$result = $qb->select($qb->func()->count('id'))
80+
->from($this->getTableName())
81+
->executeQuery();
82+
if (($cnt = $result->fetchOne()) !== false) {
83+
return (int)$cnt;
84+
}
85+
return 0;
86+
}
7387
}

‎lib/Db/QueueMapper.php‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,11 +108,14 @@ public function clearQueue(): void {
108108
/**
109109
* @throws \OCP\DB\Exception
110110
*/
111-
public function count() : int|false {
111+
public function count() : int {
112112
$qb = $this->db->getQueryBuilder();
113113
$result = $qb->select($qb->func()->count('id'))
114114
->from($this->getTableName())
115115
->executeQuery();
116-
return $result->fetchOne();
116+
if (($cnt = $result->fetchOne()) !== false) {
117+
return (int)$cnt;
118+
}
119+
return 0;
117120
}
118121
}

‎lib/Repair/AppInstallStep.php‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
<?php
2+
3+
/**
4+
* Nextcloud - ContextChat
5+
*
6+
* This file is licensed under the Affero General Public License version 3 or
7+
* later. See the COPYING file.
8+
*
9+
* @author Anupam Kumar <kyteinsky@gmail.com>
10+
* @copyright Anupam Kumar 2024
11+
*/
12+
13+
declare(strict_types=1);
14+
15+
namespace OCA\ContextChat\Repair;
16+
17+
use OCA\ContextChat\AppInfo\Application;
18+
use OCA\ContextChat\Service\ProviderConfigService;
19+
use OCP\IAppConfig;
20+
use OCP\IConfig;
21+
use OCP\Migration\IOutput;
22+
use OCP\Migration\IRepairStep;
23+
use Psr\Log\LoggerInterface;
24+
25+
class AppInstallStep implements IRepairStep {
26+
27+
public function __construct(
28+
private LoggerInterface $logger,
29+
private IAppConfig $appConfig,
30+
private IConfig $config,
31+
) {
32+
}
33+
34+
public function getName(): string {
35+
return 'Initial setup for Context Chat';
36+
}
37+
38+
/**
39+
* @param IOutput $output
40+
*/
41+
public function run(IOutput $output): void {
42+
if ($this->appConfig->getValueInt(Application::APP_ID, 'installed_time', 0, false) === 0) {
43+
$this->logger->info('Setting up Context Chat for the first time');
44+
$this->appConfig->setValueInt(Application::APP_ID, 'installed_time', time(), false);
45+
}
46+
47+
// todo: migrate to IAppConfig
48+
$providerConfigService = new ProviderConfigService($this->config);
49+
/** @psalm-suppress ArgumentTypeCoercion, UndefinedClass */
50+
$providerConfigService->updateProvider('files', 'default', '', true);
51+
}
52+
}

‎lib/Service/ActionService.php‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,4 +137,11 @@ public function updateAccessDeclSource(array $userIds, string $sourceId): void {
137137
}
138138
$this->scheduleAction(ActionType::UPDATE_ACCESS_DECL_SOURCE_ID, $payload);
139139
}
140+
141+
/**
142+
* @throws \OCP\DB\Exception
143+
*/
144+
public function count(): int {
145+
return $this->actionMapper->count();
146+
}
140147
}

‎lib/Service/QueueService.php‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,12 @@ public function getFromQueue(int $storageId, int $rootId, int $batchSize): array
6363
return $this->queueMapper->getFromQueue($storageId, $rootId, $batchSize);
6464
}
6565

66+
public function existsQueueFileId(int $fileId): bool {
67+
$queueItem = new QueueFile();
68+
$queueItem->setFileId($fileId);
69+
return $this->queueMapper->existsQueueItem($queueItem);
70+
}
71+
6672
/**
6773
* @param string $model
6874
* @param QueueFile[] $files
@@ -80,7 +86,7 @@ public function clearQueue(): void {
8086
/**
8187
* @throws \OCP\DB\Exception
8288
*/
83-
public function count(): int|false {
89+
public function count(): int {
8490
return $this->queueMapper->count();
8591
}
8692
}

0 commit comments

Comments
 (0)