Skip to content

Commit 32425b4

Browse files
marcelklehrmarcelklehr
committed
fix(StorageService#countFiles): Only count files in eligible mounts and with the correct size
and don't count encrypted files Signed-off-by: Marcel Klehr <mklehr@gmx.net>
1 parent 334c219 commit 32425b4

File tree

2 files changed

+70
-18
lines changed

2 files changed

+70
-18
lines changed

‎.github/workflows/integration-test.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -455,13 +455,13 @@ jobs:
455455
indexed = indexed + 0 # force conversion to number
456456
}
457457
END {
458-
low = total * 0.9;
459-
high = total * 1.1;
458+
low = total * 0.85;
459+
high = total * 1.15;
460460
if (indexed >= low && indexed <= high) {
461-
print "✅ Indexed files (" indexed ") are within 10% of eligible files (" total ").";
461+
print "✅ Indexed files (" indexed ") are within 15% of eligible files (" total ").";
462462
exit 0;
463463
} else {
464-
print "❌ Indexed files (" indexed ") are OUTSIDE the 10% range of eligible files (" total ").";
464+
print "❌ Indexed files (" indexed ") are OUTSIDE the 15% range of eligible files (" total ").";
465465
exit 1;
466466
}
467467
}

‎lib/Service/StorageService.php‎

Lines changed: 66 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -54,18 +54,68 @@ public function __construct(
5454
* @throws Exception
5555
*/
5656
public function countFiles(): int {
57+
$totalCount = 0;
58+
foreach ($this->getMounts() as $mount) {
59+
$totalCount += $this->countFilesInMount($mount['storage_id'], $mount['root_id']);
60+
}
61+
return $totalCount;
62+
}
63+
64+
/**
65+
* @param int $storageId
66+
* @param int $rootId
67+
* @return int
68+
*/
69+
public function countFilesInMount(int $storageId, int $rootId): int {
70+
$qb = $this->getCacheQueryBuilder();
71+
try {
72+
$qb->selectFileCache();
73+
$qb->andWhere($qb->expr()->eq('filecache.fileid', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT)));
74+
$result = $qb->executeQuery();
75+
/** @var array{path:string}|false $root */
76+
$root = $result->fetch();
77+
$result->closeCursor();
78+
} catch (Exception $e) {
79+
$this->logger->error('Could not fetch storage root', ['exception' => $e]);
80+
return 0;
81+
}
82+
83+
if ($root === false) {
84+
$this->logger->error('Could not fetch storage root');
85+
return 0;
86+
}
87+
5788
$mimeTypes = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), Application::MIMETYPES);
89+
5890
$qb = $this->getCacheQueryBuilder();
59-
$qb->select($qb->func()->count('*'))
60-
->from('filecache', 'filecache')
61-
->where($qb->expr()->in('mimetype', $qb->createNamedParameter($mimeTypes, IQueryBuilder::PARAM_INT_ARRAY)));
62-
$result = $qb->executeQuery();
63-
$count = $result->fetchOne();
91+
92+
try {
93+
$path = $root['path'] === '' ? '' : $root['path'] . '/';
94+
95+
$qb->select($qb->func()->count('*'))
96+
->from('filecache', 'filecache');
97+
$qb->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)));
98+
$qb->innerJoin('filecache', 'filecache', 'p', $qb->expr()->eq('filecache.parent', 'p.fileid'));
99+
$qb
100+
->andWhere($qb->expr()->like('filecache.path', $qb->createNamedParameter($path . '%')))
101+
->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId)))
102+
->andWhere($qb->expr()->in('filecache.mimetype', $qb->createNamedParameter($mimeTypes, IQueryBuilder::PARAM_INT_ARRAY)))
103+
->andWhere($qb->expr()->lte('filecache.size', $qb->createNamedParameter(Application::CC_MAX_SIZE, IQueryBuilder::PARAM_INT)))
104+
->andWhere($qb->expr()->gt('filecache.size', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
105+
->andWhere($qb->expr()->eq('p.encrypted', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
106+
$result = $qb->executeQuery();
107+
} catch (Exception $e) {
108+
$this->logger->error('Could not count files in mount: storage=' . $storageId . ' root=' . $rootId, ['exception' => $e]);
109+
return 0;
110+
}
111+
112+
$countInMount = $result->fetchOne();
64113
$result->closeCursor();
65-
if ($count === false) {
114+
if ($countInMount === false) {
115+
$this->logger->warning('Could not count files in mount: storage=' . $storageId . ' root=' . $rootId);
66116
return 0;
67117
}
68-
return $count;
118+
return $countInMount;
69119
}
70120

71121
/**
@@ -116,17 +166,16 @@ public function getMounts(): \Generator {
116166
/**
117167
* @param int $storageId
118168
* @param int $rootId
119-
* @param array $models
120169
* @param int $lastFileId
121170
* @param int $maxResults
122171
* @return \Generator<int,int,mixed,void>
123172
*/
124173
public function getFilesInMount(int $storageId, int $rootId, int $lastFileId = 0, int $maxResults = 100): \Generator {
125174
$qb = $this->getCacheQueryBuilder();
126175
try {
127-
$result = $qb->selectFileCache()
128-
->andWhere($qb->expr()->eq('filecache.fileid', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT)))
129-
->executeQuery();
176+
$qb->selectFileCache();
177+
$qb->andWhere($qb->expr()->eq('filecache.fileid', $qb->createNamedParameter($rootId, IQueryBuilder::PARAM_INT)));
178+
$result = $qb->executeQuery();
130179
/** @var array{path:string}|false $root */
131180
$root = $result->fetch();
132181
$result->closeCursor();
@@ -147,12 +196,15 @@ public function getFilesInMount(int $storageId, int $rootId, int $lastFileId = 0
147196
try {
148197
$path = $root['path'] === '' ? '' : $root['path'] . '/';
149198

150-
$qb->selectFileCache()
151-
->whereStorageId($storageId)
199+
$qb->selectFileCache();
200+
$qb->whereStorageId($storageId);
201+
$qb->innerJoin('filecache', 'filecache', 'p', $qb->expr()->eq('filecache.parent', 'p.id'));
202+
$qb
152203
->andWhere($qb->expr()->like('path', $qb->createNamedParameter($path . '%')))
153204
->andWhere($qb->expr()->eq('storage', $qb->createNamedParameter($storageId)))
154205
->andWhere($qb->expr()->gt('filecache.fileid', $qb->createNamedParameter($lastFileId)))
155-
->andWhere($qb->expr()->in('mimetype', $qb->createNamedParameter($mimeTypes, IQueryBuilder::PARAM_INT_ARRAY)));
206+
->andWhere($qb->expr()->in('mimetype', $qb->createNamedParameter($mimeTypes, IQueryBuilder::PARAM_INT_ARRAY)))
207+
->andWhere($qb->expr()->eq('p.encrypted', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
156208

157209
if ($maxResults !== 0) {
158210
$qb->setMaxResults($maxResults);

0 commit comments

Comments
 (0)