Skip to content

Commit 874c283

Browse files
icewind1991icewind1991
authored
Merge pull request #51603 from nextcloud/object-store-orphan
Add command to list orphan objects
2 parents ccaa463 + 7ce06f4 commit 874c283

File tree

12 files changed

+370
-3
lines changed

12 files changed

+370
-3
lines changed

‎apps/files/appinfo/info.xml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@
4949
<command>OCA\Files\Command\Object\Delete</command>
5050
<command>OCA\Files\Command\Object\Get</command>
5151
<command>OCA\Files\Command\Object\Put</command>
52+
<command>OCA\Files\Command\Object\Info</command>
53+
<command>OCA\Files\Command\Object\ListObject</command>
54+
<command>OCA\Files\Command\Object\Orphans</command>
5255
</commands>
5356

5457
<settings>

‎apps/files/composer/composer/autoload_classmap.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@
3535
'OCA\\Files\\Command\\Move' => $baseDir . '/../lib/Command/Move.php',
3636
'OCA\\Files\\Command\\Object\\Delete' => $baseDir . '/../lib/Command/Object/Delete.php',
3737
'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php',
38+
'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php',
39+
'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php',
3840
'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php',
41+
'OCA\\Files\\Command\\Object\\Orphans' => $baseDir . '/../lib/Command/Object/Orphans.php',
3942
'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php',
4043
'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php',
4144
'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php',

‎apps/files/composer/composer/autoload_static.php‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,10 @@ class ComposerStaticInitFiles
5050
'OCA\\Files\\Command\\Move' => __DIR__ . '/..' . '/../lib/Command/Move.php',
5151
'OCA\\Files\\Command\\Object\\Delete' => __DIR__ . '/..' . '/../lib/Command/Object/Delete.php',
5252
'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php',
53+
'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php',
54+
'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php',
5355
'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php',
56+
'OCA\\Files\\Command\\Object\\Orphans' => __DIR__ . '/..' . '/../lib/Command/Object/Orphans.php',
5457
'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php',
5558
'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php',
5659
'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php',

‎apps/files/lib/Command/Object/Info.php‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
6+
* SPDX-License-Identifier: AGPL-3.0-or-later
7+
*/
8+
9+
namespace OCA\Files\Command\Object;
10+
11+
use OC\Core\Command\Base;
12+
use OCP\Files\IMimeTypeDetector;
13+
use OCP\Files\ObjectStore\IObjectStoreMetaData;
14+
use OCP\Util;
15+
use Symfony\Component\Console\Input\InputArgument;
16+
use Symfony\Component\Console\Input\InputInterface;
17+
use Symfony\Component\Console\Input\InputOption;
18+
use Symfony\Component\Console\Output\OutputInterface;
19+
20+
class Info extends Base {
21+
public function __construct(
22+
private ObjectUtil $objectUtils,
23+
private IMimeTypeDetector $mimeTypeDetector,
24+
) {
25+
parent::__construct();
26+
}
27+
28+
protected function configure(): void {
29+
parent::configure();
30+
$this
31+
->setName('files:object:info')
32+
->setDescription('Get the metadata of an object')
33+
->addArgument('object', InputArgument::REQUIRED, 'Object to get')
34+
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to get the object from, only required in cases where it can't be determined from the config");
35+
}
36+
37+
public function execute(InputInterface $input, OutputInterface $output): int {
38+
$object = $input->getArgument('object');
39+
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
40+
if (!$objectStore) {
41+
return self::FAILURE;
42+
}
43+
44+
if (!$objectStore instanceof IObjectStoreMetaData) {
45+
$output->writeln('<error>Configured object store does currently not support retrieve metadata</error>');
46+
return self::FAILURE;
47+
}
48+
49+
if (!$objectStore->objectExists($object)) {
50+
$output->writeln("<error>Object $object does not exist</error>");
51+
return self::FAILURE;
52+
}
53+
54+
try {
55+
$meta = $objectStore->getObjectMetaData($object);
56+
} catch (\Exception $e) {
57+
$msg = $e->getMessage();
58+
$output->writeln("<error>Failed to read $object from object store: $msg</error>");
59+
return self::FAILURE;
60+
}
61+
62+
if ($input->getOption('output') === 'plain' && isset($meta['size'])) {
63+
$meta['size'] = Util::humanFileSize($meta['size']);
64+
}
65+
if (isset($meta['mtime'])) {
66+
$meta['mtime'] = $meta['mtime']->format(\DateTimeImmutable::ATOM);
67+
}
68+
if (!isset($meta['mimetype'])) {
69+
$handle = $objectStore->readObject($object);
70+
$head = fread($handle, 8192);
71+
fclose($handle);
72+
$meta['mimetype'] = $this->mimeTypeDetector->detectString($head);
73+
}
74+
75+
$this->writeArrayInOutputFormat($input, $output, $meta);
76+
77+
return self::SUCCESS;
78+
}
79+
80+
}

‎apps/files/lib/Command/Object/ListObject.php‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
6+
* SPDX-License-Identifier: AGPL-3.0-or-later
7+
*/
8+
9+
namespace OCA\Files\Command\Object;
10+
11+
use OC\Core\Command\Base;
12+
use OCP\Files\ObjectStore\IObjectStoreMetaData;
13+
use Symfony\Component\Console\Input\InputInterface;
14+
use Symfony\Component\Console\Input\InputOption;
15+
use Symfony\Component\Console\Output\OutputInterface;
16+
17+
class ListObject extends Base {
18+
private const CHUNK_SIZE = 100;
19+
20+
public function __construct(
21+
private readonly ObjectUtil $objectUtils,
22+
) {
23+
parent::__construct();
24+
}
25+
26+
protected function configure(): void {
27+
parent::configure();
28+
$this
29+
->setName('files:object:list')
30+
->setDescription('List all objects in the object store')
31+
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
32+
}
33+
34+
public function execute(InputInterface $input, OutputInterface $output): int {
35+
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
36+
if (!$objectStore) {
37+
return self::FAILURE;
38+
}
39+
40+
if (!$objectStore instanceof IObjectStoreMetaData) {
41+
$output->writeln('<error>Configured object store does currently not support listing objects</error>');
42+
return self::FAILURE;
43+
}
44+
$objects = $objectStore->listObjects();
45+
$objects = $this->objectUtils->formatObjects($objects, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN);
46+
$this->writeStreamingTableInOutputFormat($input, $output, $objects, self::CHUNK_SIZE);
47+
48+
return self::SUCCESS;
49+
}
50+
}

‎apps/files/lib/Command/Object/ObjectUtil.php‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
use OCP\Files\ObjectStore\IObjectStore;
1313
use OCP\IConfig;
1414
use OCP\IDBConnection;
15+
use OCP\Util;
1516
use Symfony\Component\Console\Output\OutputInterface;
1617

1718
class ObjectUtil {
@@ -91,4 +92,24 @@ public function objectExistsInDb(string $object): int|false {
9192

9293
return $fileId;
9394
}
95+
96+
public function formatObjects(\Iterator $objects, bool $humanOutput): \Iterator {
97+
foreach ($objects as $object) {
98+
yield $this->formatObject($object, $humanOutput);
99+
}
100+
}
101+
102+
public function formatObject(array $object, bool $humanOutput): array {
103+
$row = array_merge([
104+
'urn' => $object['urn'],
105+
], ($object['metadata'] ?? []));
106+
107+
if ($humanOutput && isset($row['size'])) {
108+
$row['size'] = Util::humanFileSize($row['size']);
109+
}
110+
if (isset($row['mtime'])) {
111+
$row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM);
112+
}
113+
return $row;
114+
}
94115
}

‎apps/files/lib/Command/Object/Orphans.php‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
6+
* SPDX-License-Identifier: AGPL-3.0-or-later
7+
*/
8+
9+
namespace OCA\Files\Command\Object;
10+
11+
use OC\Core\Command\Base;
12+
use OCP\DB\QueryBuilder\IQueryBuilder;
13+
use OCP\Files\ObjectStore\IObjectStoreMetaData;
14+
use OCP\IDBConnection;
15+
use Symfony\Component\Console\Input\InputInterface;
16+
use Symfony\Component\Console\Input\InputOption;
17+
use Symfony\Component\Console\Output\OutputInterface;
18+
19+
class Orphans extends Base {
20+
private const CHUNK_SIZE = 100;
21+
22+
private ?IQueryBuilder $query = null;
23+
24+
public function __construct(
25+
private readonly ObjectUtil $objectUtils,
26+
private readonly IDBConnection $connection,
27+
) {
28+
parent::__construct();
29+
}
30+
31+
private function getQuery(): IQueryBuilder {
32+
if (!$this->query) {
33+
$this->query = $this->connection->getQueryBuilder();
34+
$this->query->select('fileid')
35+
->from('filecache')
36+
->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id')));
37+
}
38+
return $this->query;
39+
}
40+
41+
protected function configure(): void {
42+
parent::configure();
43+
$this
44+
->setName('files:object:orphans')
45+
->setDescription('List all objects in the object store that don\'t have a matching entry in the database')
46+
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
47+
}
48+
49+
public function execute(InputInterface $input, OutputInterface $output): int {
50+
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
51+
if (!$objectStore) {
52+
return self::FAILURE;
53+
}
54+
55+
if (!$objectStore instanceof IObjectStoreMetaData) {
56+
$output->writeln('<error>Configured object store does currently not support listing objects</error>');
57+
return self::FAILURE;
58+
}
59+
$prefixLength = strlen('urn:oid:');
60+
61+
$objects = $objectStore->listObjects('urn:oid:');
62+
$orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) {
63+
$fileId = (int)substr($object['urn'], $prefixLength);
64+
return !$this->fileIdInDb($fileId);
65+
});
66+
67+
$orphans = $this->objectUtils->formatObjects($orphans, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN);
68+
$this->writeStreamingTableInOutputFormat($input, $output, $orphans, self::CHUNK_SIZE);
69+
70+
return self::SUCCESS;
71+
}
72+
73+
private function fileIdInDb(int $fileId): bool {
74+
$query = $this->getQuery();
75+
$query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT);
76+
$result = $query->executeQuery();
77+
return $result->fetchOne() !== false;
78+
}
79+
}

‎core/Command/Base.php‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,58 @@ protected function writeTableInOutputFormat(InputInterface $input, OutputInterfa
8888
}
8989
}
9090

91+
protected function writeStreamingTableInOutputFormat(InputInterface $input, OutputInterface $output, \Iterator $items, int $tableGroupSize): void {
92+
switch ($input->getOption('output')) {
93+
case self::OUTPUT_FORMAT_JSON:
94+
case self::OUTPUT_FORMAT_JSON_PRETTY:
95+
$this->writeStreamingJsonArray($input, $output, $items);
96+
break;
97+
default:
98+
foreach ($this->chunkIterator($items, $tableGroupSize) as $chunk) {
99+
$this->writeTableInOutputFormat($input, $output, $chunk);
100+
}
101+
break;
102+
}
103+
}
104+
105+
protected function writeStreamingJsonArray(InputInterface $input, OutputInterface $output, \Iterator $items): void {
106+
$first = true;
107+
$outputType = $input->getOption('output');
108+
109+
$output->writeln('[');
110+
foreach ($items as $item) {
111+
if (!$first) {
112+
$output->writeln(',');
113+
}
114+
if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) {
115+
$output->write(json_encode($item, JSON_PRETTY_PRINT));
116+
} else {
117+
$output->write(json_encode($item));
118+
}
119+
$first = false;
120+
}
121+
$output->writeln("\n]");
122+
}
123+
124+
public function chunkIterator(\Iterator $iterator, int $count): \Iterator {
125+
$chunk = [];
126+
127+
for ($i = 0; $iterator->valid(); $i++) {
128+
$chunk[] = $iterator->current();
129+
$iterator->next();
130+
if (count($chunk) == $count) {
131+
// Got a full chunk, yield and start a new one
132+
yield $chunk;
133+
$chunk = [];
134+
}
135+
}
136+
137+
if (count($chunk)) {
138+
// Yield the last chunk even if incomplete
139+
yield $chunk;
140+
}
141+
}
142+
91143

92144
/**
93145
* @param mixed $item

‎lib/composer/composer/autoload_classmap.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -456,6 +456,7 @@
456456
'OCP\\Files\\Notify\\INotifyHandler' => $baseDir . '/lib/public/Files/Notify/INotifyHandler.php',
457457
'OCP\\Files\\Notify\\IRenameChange' => $baseDir . '/lib/public/Files/Notify/IRenameChange.php',
458458
'OCP\\Files\\ObjectStore\\IObjectStore' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStore.php',
459+
'OCP\\Files\\ObjectStore\\IObjectStoreMetaData' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMetaData.php',
459460
'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php',
460461
'OCP\\Files\\ReservedWordException' => $baseDir . '/lib/public/Files/ReservedWordException.php',
461462
'OCP\\Files\\Search\\ISearchBinaryOperator' => $baseDir . '/lib/public/Files/Search/ISearchBinaryOperator.php',

‎lib/composer/composer/autoload_static.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -505,6 +505,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
505505
'OCP\\Files\\Notify\\INotifyHandler' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/INotifyHandler.php',
506506
'OCP\\Files\\Notify\\IRenameChange' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/IRenameChange.php',
507507
'OCP\\Files\\ObjectStore\\IObjectStore' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStore.php',
508+
'OCP\\Files\\ObjectStore\\IObjectStoreMetaData' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMetaData.php',
508509
'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php',
509510
'OCP\\Files\\ReservedWordException' => __DIR__ . '/../../..' . '/lib/public/Files/ReservedWordException.php',
510511
'OCP\\Files\\Search\\ISearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/public/Files/Search/ISearchBinaryOperator.php',

0 commit comments

Comments
 (0)