Skip to content

Commit 18e9980

Browse files
icewind1991icewind1991
committed
feat: add command to list orphan objects
Signed-off-by: Robin Appelman <[email protected]>
1 parent 7465298 commit 18e9980

File tree

6 files changed

+154
-72
lines changed

6 files changed

+154
-72
lines changed

‎apps/files/appinfo/info.xml

+1
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@
5151
<command>OCA\Files\Command\Object\Put</command>
5252
<command>OCA\Files\Command\Object\Info</command>
5353
<command>OCA\Files\Command\Object\ListObject</command>
54+
<command>OCA\Files\Command\Object\Orphans</command>
5455
</commands>
5556

5657
<settings>

‎apps/files/composer/composer/autoload_classmap.php

+1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php',
3838
'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php',
3939
'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php',
40+
'OCA\\Files\\Command\\Object\\Orphans' => $baseDir . '/../lib/Command/Object/Orphans.php',
4041
'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php',
4142
'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php',
4243
'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php',

‎apps/files/composer/composer/autoload_static.php

+1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ class ComposerStaticInitFiles
5252
'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php',
5353
'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php',
5454
'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php',
55+
'OCA\\Files\\Command\\Object\\Orphans' => __DIR__ . '/..' . '/../lib/Command/Object/Orphans.php',
5556
'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php',
5657
'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php',
5758
'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php',

‎apps/files/lib/Command/Object/ListObject.php

+1-71
Original file line numberDiff line numberDiff line change
@@ -41,79 +41,9 @@ public function execute(InputInterface $input, OutputInterface $output): int {
4141
$output->writeln('<error>Configured object store does currently not support listing objects</error>');
4242
return self::FAILURE;
4343
}
44-
$outputType = $input->getOption('output');
45-
$humanOutput = $outputType === self::OUTPUT_FORMAT_PLAIN;
46-
47-
if (!$humanOutput) {
48-
$output->writeln('[');
49-
}
5044
$objects = $objectStore->listObjects();
51-
$first = true;
52-
53-
foreach ($this->chunkIterator($objects, self::CHUNK_SIZE) as $chunk) {
54-
if ($outputType === self::OUTPUT_FORMAT_PLAIN) {
55-
$this->outputChunk($input, $output, $chunk);
56-
} else {
57-
foreach ($chunk as $object) {
58-
if (!$first) {
59-
$output->writeln(',');
60-
}
61-
$row = $this->formatObject($object, $humanOutput);
62-
if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) {
63-
$output->write(json_encode($row, JSON_PRETTY_PRINT));
64-
} else {
65-
$output->write(json_encode($row));
66-
}
67-
$first = false;
68-
}
69-
}
70-
}
71-
72-
if (!$humanOutput) {
73-
$output->writeln("\n]");
74-
}
45+
$this->objectUtils->writeIteratorToOutput($input, $output, $objects, self::CHUNK_SIZE);
7546

7647
return self::SUCCESS;
7748
}
78-
79-
private function formatObject(array $object, bool $humanOutput): array {
80-
$row = array_merge([
81-
'urn' => $object['urn'],
82-
], ($object['metadata'] ?? []));
83-
84-
if ($humanOutput && isset($row['size'])) {
85-
$row['size'] = \OC_Helper::humanFileSize($row['size']);
86-
}
87-
if (isset($row['mtime'])) {
88-
$row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM);
89-
}
90-
return $row;
91-
}
92-
93-
private function outputChunk(InputInterface $input, OutputInterface $output, iterable $chunk): void {
94-
$result = [];
95-
$humanOutput = $input->getOption('output') === "plain";
96-
97-
foreach ($chunk as $object) {
98-
$result[] = $this->formatObject($object, $humanOutput);
99-
}
100-
$this->writeTableInOutputFormat($input, $output, $result);
101-
}
102-
103-
function chunkIterator(\Iterator $iterator, int $count): \Iterator {
104-
$chunk = [];
105-
106-
for($i = 0; $iterator->valid(); $i++){
107-
$chunk[] = $iterator->current();
108-
$iterator->next();
109-
if(count($chunk) == $count){
110-
yield $chunk;
111-
$chunk = [];
112-
}
113-
}
114-
115-
if(count($chunk)){
116-
yield $chunk;
117-
}
118-
}
11949
}

‎apps/files/lib/Command/Object/ObjectUtil.php

+77-1
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,15 @@
88

99
namespace OCA\Files\Command\Object;
1010

11+
use OC\Core\Command\Base;
1112
use OCP\DB\QueryBuilder\IQueryBuilder;
1213
use OCP\Files\ObjectStore\IObjectStore;
1314
use OCP\IConfig;
1415
use OCP\IDBConnection;
16+
use Symfony\Component\Console\Input\InputInterface;
1517
use Symfony\Component\Console\Output\OutputInterface;
1618

17-
class ObjectUtil {
19+
class ObjectUtil extends Base {
1820
public function __construct(
1921
private IConfig $config,
2022
private IDBConnection $connection,
@@ -91,4 +93,78 @@ public function objectExistsInDb(string $object): int|false {
9193

9294
return $fileId;
9395
}
96+
97+
public function writeIteratorToOutput(InputInterface $input, OutputInterface $output, \Iterator $objects, int $chunkSize): void {
98+
$outputType = $input->getOption('output');
99+
$humanOutput = $outputType === Base::OUTPUT_FORMAT_PLAIN;
100+
$first = true;
101+
102+
if (!$humanOutput) {
103+
$output->writeln('[');
104+
}
105+
106+
foreach ($this->chunkIterator($objects, $chunkSize) as $chunk) {
107+
if ($outputType === Base::OUTPUT_FORMAT_PLAIN) {
108+
$this->outputChunk($input, $output, $chunk);
109+
} else {
110+
foreach ($chunk as $object) {
111+
if (!$first) {
112+
$output->writeln(',');
113+
}
114+
$row = $this->formatObject($object, $humanOutput);
115+
if ($outputType === Base::OUTPUT_FORMAT_JSON_PRETTY) {
116+
$output->write(json_encode($row, JSON_PRETTY_PRINT));
117+
} else {
118+
$output->write(json_encode($row));
119+
}
120+
$first = false;
121+
}
122+
}
123+
}
124+
125+
if (!$humanOutput) {
126+
$output->writeln("\n]");
127+
}
128+
}
129+
130+
private function formatObject(array $object, bool $humanOutput): array {
131+
$row = array_merge([
132+
'urn' => $object['urn'],
133+
], ($object['metadata'] ?? []));
134+
135+
if ($humanOutput && isset($row['size'])) {
136+
$row['size'] = \OC_Helper::humanFileSize($row['size']);
137+
}
138+
if (isset($row['mtime'])) {
139+
$row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM);
140+
}
141+
return $row;
142+
}
143+
144+
private function outputChunk(InputInterface $input, OutputInterface $output, iterable $chunk): void {
145+
$result = [];
146+
$humanOutput = $input->getOption('output') === 'plain';
147+
148+
foreach ($chunk as $object) {
149+
$result[] = $this->formatObject($object, $humanOutput);
150+
}
151+
$this->writeTableInOutputFormat($input, $output, $result);
152+
}
153+
154+
public function chunkIterator(\Iterator $iterator, int $count): \Iterator {
155+
$chunk = [];
156+
157+
for ($i = 0; $iterator->valid(); $i++) {
158+
$chunk[] = $iterator->current();
159+
$iterator->next();
160+
if (count($chunk) == $count) {
161+
yield $chunk;
162+
$chunk = [];
163+
}
164+
}
165+
166+
if (count($chunk)) {
167+
yield $chunk;
168+
}
169+
}
94170
}

‎apps/files/lib/Command/Object/Orphans.php

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
/**
5+
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
6+
* SPDX-License-Identifier: AGPL-3.0-or-later
7+
*/
8+
9+
namespace OCA\Files\Command\Object;
10+
11+
use OC\Core\Command\Base;
12+
use OCP\DB\QueryBuilder\IQueryBuilder;
13+
use OCP\Files\ObjectStore\IObjectStoreMetaData;
14+
use OCP\IDBConnection;
15+
use Symfony\Component\Console\Input\InputInterface;
16+
use Symfony\Component\Console\Input\InputOption;
17+
use Symfony\Component\Console\Output\OutputInterface;
18+
19+
class Orphans extends Base {
20+
private const CHUNK_SIZE = 100;
21+
22+
private IQueryBuilder $query;
23+
24+
public function __construct(
25+
private readonly ObjectUtil $objectUtils,
26+
IDBConnection $connection,
27+
) {
28+
parent::__construct();
29+
30+
$this->query = $connection->getQueryBuilder();
31+
$this->query->select('fileid')
32+
->from('filecache')
33+
->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id')));
34+
}
35+
36+
protected function configure(): void {
37+
parent::configure();
38+
$this
39+
->setName('files:object:orphans')
40+
->setDescription('List all objects in the object store that don\'t have a matching entry in the database')
41+
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
42+
}
43+
44+
public function execute(InputInterface $input, OutputInterface $output): int {
45+
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
46+
if (!$objectStore) {
47+
return self::FAILURE;
48+
}
49+
50+
if (!$objectStore instanceof IObjectStoreMetaData) {
51+
$output->writeln('<error>Configured object store does currently not support listing objects</error>');
52+
return self::FAILURE;
53+
}
54+
$prefixLength = strlen('urn:oid:');
55+
56+
$objects = $objectStore->listObjects('urn:oid:');
57+
$objects->rewind();
58+
$orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) {
59+
$fileId = (int)substr($object['urn'], $prefixLength);
60+
return !$this->fileIdInDb($fileId);
61+
});
62+
$orphans = new \ArrayIterator(iterator_to_array($orphans));
63+
$this->objectUtils->writeIteratorToOutput($input, $output, $orphans, self::CHUNK_SIZE);
64+
65+
return self::SUCCESS;
66+
}
67+
68+
private function fileIdInDb(int $fileId): bool {
69+
$this->query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT);
70+
$result = $this->query->executeQuery();
71+
return $result->fetchOne() !== false;
72+
}
73+
}

0 commit comments

Comments
 (0)