diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml
index cdcc79e47fb76..95f6153e1d710 100644
--- a/apps/files/appinfo/info.xml
+++ b/apps/files/appinfo/info.xml
@@ -49,6 +49,9 @@
OCA\Files\Command\Object\Delete
OCA\Files\Command\Object\Get
OCA\Files\Command\Object\Put
+ OCA\Files\Command\Object\Info
+ OCA\Files\Command\Object\ListObject
+ OCA\Files\Command\Object\Orphans
diff --git a/apps/files/composer/composer/autoload_classmap.php b/apps/files/composer/composer/autoload_classmap.php
index 0d9e6aa2d77dd..a74df7ba3d24f 100644
--- a/apps/files/composer/composer/autoload_classmap.php
+++ b/apps/files/composer/composer/autoload_classmap.php
@@ -35,7 +35,10 @@
'OCA\\Files\\Command\\Move' => $baseDir . '/../lib/Command/Move.php',
'OCA\\Files\\Command\\Object\\Delete' => $baseDir . '/../lib/Command/Object/Delete.php',
'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php',
+ 'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php',
+ 'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php',
'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php',
+ 'OCA\\Files\\Command\\Object\\Orphans' => $baseDir . '/../lib/Command/Object/Orphans.php',
'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php',
'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php',
'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php',
diff --git a/apps/files/composer/composer/autoload_static.php b/apps/files/composer/composer/autoload_static.php
index 5ece90731783b..1d79f38e35a9a 100644
--- a/apps/files/composer/composer/autoload_static.php
+++ b/apps/files/composer/composer/autoload_static.php
@@ -50,7 +50,10 @@ class ComposerStaticInitFiles
'OCA\\Files\\Command\\Move' => __DIR__ . '/..' . '/../lib/Command/Move.php',
'OCA\\Files\\Command\\Object\\Delete' => __DIR__ . '/..' . '/../lib/Command/Object/Delete.php',
'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php',
+ 'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php',
+ 'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php',
'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php',
+ 'OCA\\Files\\Command\\Object\\Orphans' => __DIR__ . '/..' . '/../lib/Command/Object/Orphans.php',
'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php',
'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php',
'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php',
diff --git a/apps/files/lib/Command/Object/Info.php b/apps/files/lib/Command/Object/Info.php
new file mode 100644
index 0000000000000..6748de37cfe2b
--- /dev/null
+++ b/apps/files/lib/Command/Object/Info.php
@@ -0,0 +1,80 @@
+setName('files:object:info')
+ ->setDescription('Get the metadata of an object')
+ ->addArgument('object', InputArgument::REQUIRED, 'Object to get')
+ ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to get the object from, only required in cases where it can't be determined from the config");
+ }
+
+ public function execute(InputInterface $input, OutputInterface $output): int {
+ $object = $input->getArgument('object');
+ $objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
+ if (!$objectStore) {
+ return self::FAILURE;
+ }
+
+ if (!$objectStore instanceof IObjectStoreMetaData) {
+ $output->writeln('Configured object store does currently not support retrieve metadata');
+ return self::FAILURE;
+ }
+
+ if (!$objectStore->objectExists($object)) {
+ $output->writeln("Object $object does not exist");
+ return self::FAILURE;
+ }
+
+ try {
+ $meta = $objectStore->getObjectMetaData($object);
+ } catch (\Exception $e) {
+ $msg = $e->getMessage();
+ $output->writeln("Failed to read $object from object store: $msg");
+ return self::FAILURE;
+ }
+
+ if ($input->getOption('output') === 'plain' && isset($meta['size'])) {
+ $meta['size'] = Util::humanFileSize($meta['size']);
+ }
+ if (isset($meta['mtime'])) {
+ $meta['mtime'] = $meta['mtime']->format(\DateTimeImmutable::ATOM);
+ }
+ if (!isset($meta['mimetype'])) {
+ $handle = $objectStore->readObject($object);
+ $head = fread($handle, 8192);
+ fclose($handle);
+ $meta['mimetype'] = $this->mimeTypeDetector->detectString($head);
+ }
+
+ $this->writeArrayInOutputFormat($input, $output, $meta);
+
+ return self::SUCCESS;
+ }
+
+}
diff --git a/apps/files/lib/Command/Object/ListObject.php b/apps/files/lib/Command/Object/ListObject.php
new file mode 100644
index 0000000000000..5d30232e09f8f
--- /dev/null
+++ b/apps/files/lib/Command/Object/ListObject.php
@@ -0,0 +1,50 @@
+setName('files:object:list')
+ ->setDescription('List all objects in the object store')
+ ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
+ }
+
+ public function execute(InputInterface $input, OutputInterface $output): int {
+ $objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
+ if (!$objectStore) {
+ return self::FAILURE;
+ }
+
+ if (!$objectStore instanceof IObjectStoreMetaData) {
+ $output->writeln('Configured object store does currently not support listing objects');
+ return self::FAILURE;
+ }
+ $objects = $objectStore->listObjects();
+ $objects = $this->objectUtils->formatObjects($objects, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN);
+ $this->writeStreamingTableInOutputFormat($input, $output, $objects, self::CHUNK_SIZE);
+
+ return self::SUCCESS;
+ }
+}
diff --git a/apps/files/lib/Command/Object/ObjectUtil.php b/apps/files/lib/Command/Object/ObjectUtil.php
index c4ab59608fbac..5f053c2c42fff 100644
--- a/apps/files/lib/Command/Object/ObjectUtil.php
+++ b/apps/files/lib/Command/Object/ObjectUtil.php
@@ -12,6 +12,7 @@
use OCP\Files\ObjectStore\IObjectStore;
use OCP\IConfig;
use OCP\IDBConnection;
+use OCP\Util;
use Symfony\Component\Console\Output\OutputInterface;
class ObjectUtil {
@@ -91,4 +92,24 @@ public function objectExistsInDb(string $object): int|false {
return $fileId;
}
+
+ public function formatObjects(\Iterator $objects, bool $humanOutput): \Iterator {
+ foreach ($objects as $object) {
+ yield $this->formatObject($object, $humanOutput);
+ }
+ }
+
+ public function formatObject(array $object, bool $humanOutput): array {
+ $row = array_merge([
+ 'urn' => $object['urn'],
+ ], ($object['metadata'] ?? []));
+
+ if ($humanOutput && isset($row['size'])) {
+ $row['size'] = Util::humanFileSize($row['size']);
+ }
+ if (isset($row['mtime'])) {
+ $row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM);
+ }
+ return $row;
+ }
}
diff --git a/apps/files/lib/Command/Object/Orphans.php b/apps/files/lib/Command/Object/Orphans.php
new file mode 100644
index 0000000000000..f7132540fc82f
--- /dev/null
+++ b/apps/files/lib/Command/Object/Orphans.php
@@ -0,0 +1,79 @@
+query) {
+ $this->query = $this->connection->getQueryBuilder();
+ $this->query->select('fileid')
+ ->from('filecache')
+ ->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id')));
+ }
+ return $this->query;
+ }
+
+ protected function configure(): void {
+ parent::configure();
+ $this
+ ->setName('files:object:orphans')
+ ->setDescription('List all objects in the object store that don\'t have a matching entry in the database')
+ ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
+ }
+
+ public function execute(InputInterface $input, OutputInterface $output): int {
+ $objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
+ if (!$objectStore) {
+ return self::FAILURE;
+ }
+
+ if (!$objectStore instanceof IObjectStoreMetaData) {
+ $output->writeln('Configured object store does currently not support listing objects');
+ return self::FAILURE;
+ }
+ $prefixLength = strlen('urn:oid:');
+
+ $objects = $objectStore->listObjects('urn:oid:');
+ $orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) {
+ $fileId = (int)substr($object['urn'], $prefixLength);
+ return !$this->fileIdInDb($fileId);
+ });
+
+ $orphans = $this->objectUtils->formatObjects($orphans, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN);
+ $this->writeStreamingTableInOutputFormat($input, $output, $orphans, self::CHUNK_SIZE);
+
+ return self::SUCCESS;
+ }
+
+ private function fileIdInDb(int $fileId): bool {
+ $query = $this->getQuery();
+ $query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT);
+ $result = $query->executeQuery();
+ return $result->fetchOne() !== false;
+ }
+}
diff --git a/core/Command/Base.php b/core/Command/Base.php
index b915ae2ae4a3a..c9b6337b64a9a 100644
--- a/core/Command/Base.php
+++ b/core/Command/Base.php
@@ -88,6 +88,58 @@ protected function writeTableInOutputFormat(InputInterface $input, OutputInterfa
}
}
+ protected function writeStreamingTableInOutputFormat(InputInterface $input, OutputInterface $output, \Iterator $items, int $tableGroupSize): void {
+ switch ($input->getOption('output')) {
+ case self::OUTPUT_FORMAT_JSON:
+ case self::OUTPUT_FORMAT_JSON_PRETTY:
+ $this->writeStreamingJsonArray($input, $output, $items);
+ break;
+ default:
+ foreach ($this->chunkIterator($items, $tableGroupSize) as $chunk) {
+ $this->writeTableInOutputFormat($input, $output, $chunk);
+ }
+ break;
+ }
+ }
+
+ protected function writeStreamingJsonArray(InputInterface $input, OutputInterface $output, \Iterator $items): void {
+ $first = true;
+ $outputType = $input->getOption('output');
+
+ $output->writeln('[');
+ foreach ($items as $item) {
+ if (!$first) {
+ $output->writeln(',');
+ }
+ if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) {
+ $output->write(json_encode($item, JSON_PRETTY_PRINT));
+ } else {
+ $output->write(json_encode($item));
+ }
+ $first = false;
+ }
+ $output->writeln("\n]");
+ }
+
+ public function chunkIterator(\Iterator $iterator, int $count): \Iterator {
+ $chunk = [];
+
+ for ($i = 0; $iterator->valid(); $i++) {
+ $chunk[] = $iterator->current();
+ $iterator->next();
+ if (count($chunk) == $count) {
+ // Got a full chunk, yield and start a new one
+ yield $chunk;
+ $chunk = [];
+ }
+ }
+
+ if (count($chunk)) {
+ // Yield the last chunk even if incomplete
+ yield $chunk;
+ }
+ }
+
/**
* @param mixed $item
diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php
index 2d0b62d749f7f..52e3075e4134b 100644
--- a/lib/composer/composer/autoload_classmap.php
+++ b/lib/composer/composer/autoload_classmap.php
@@ -456,6 +456,7 @@
'OCP\\Files\\Notify\\INotifyHandler' => $baseDir . '/lib/public/Files/Notify/INotifyHandler.php',
'OCP\\Files\\Notify\\IRenameChange' => $baseDir . '/lib/public/Files/Notify/IRenameChange.php',
'OCP\\Files\\ObjectStore\\IObjectStore' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStore.php',
+ 'OCP\\Files\\ObjectStore\\IObjectStoreMetaData' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMetaData.php',
'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php',
'OCP\\Files\\ReservedWordException' => $baseDir . '/lib/public/Files/ReservedWordException.php',
'OCP\\Files\\Search\\ISearchBinaryOperator' => $baseDir . '/lib/public/Files/Search/ISearchBinaryOperator.php',
diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php
index 64210bd584344..e98bc3e1aaaee 100644
--- a/lib/composer/composer/autoload_static.php
+++ b/lib/composer/composer/autoload_static.php
@@ -505,6 +505,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
'OCP\\Files\\Notify\\INotifyHandler' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/INotifyHandler.php',
'OCP\\Files\\Notify\\IRenameChange' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/IRenameChange.php',
'OCP\\Files\\ObjectStore\\IObjectStore' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStore.php',
+ 'OCP\\Files\\ObjectStore\\IObjectStoreMetaData' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMetaData.php',
'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php',
'OCP\\Files\\ReservedWordException' => __DIR__ . '/../../..' . '/lib/public/Files/ReservedWordException.php',
'OCP\\Files\\Search\\ISearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/public/Files/Search/ISearchBinaryOperator.php',
diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php
index 41ab75caf45fe..e970fb6ac145a 100644
--- a/lib/private/Files/ObjectStore/S3.php
+++ b/lib/private/Files/ObjectStore/S3.php
@@ -3,14 +3,16 @@
* SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
+
namespace OC\Files\ObjectStore;
use Aws\Result;
use Exception;
use OCP\Files\ObjectStore\IObjectStore;
+use OCP\Files\ObjectStore\IObjectStoreMetaData;
use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
-class S3 implements IObjectStore, IObjectStoreMultiPartUpload {
+class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaData {
use S3ConnectionTrait;
use S3ObjectTrait;
@@ -61,7 +63,7 @@ public function getMultipartUploads(string $urn, string $uploadId): array {
'Key' => $urn,
'UploadId' => $uploadId,
'MaxParts' => 1000,
- 'PartNumberMarker' => $partNumberMarker
+ 'PartNumberMarker' => $partNumberMarker,
] + $this->getSSECParameters());
$parts = array_merge($parts, $result->get('Parts') ?? []);
$isTruncated = $result->get('IsTruncated');
@@ -89,7 +91,41 @@ public function abortMultipartUpload($urn, $uploadId): void {
$this->getConnection()->abortMultipartUpload([
'Bucket' => $this->bucket,
'Key' => $urn,
- 'UploadId' => $uploadId
+ 'UploadId' => $uploadId,
]);
}
+
+ public function getObjectMetaData(string $urn): array {
+ $object = $this->getConnection()->headObject([
+ 'Bucket' => $this->bucket,
+ 'Key' => $urn
+ ] + $this->getSSECParameters())->toArray();
+ return [
+ 'mtime' => $object['LastModified'],
+ 'etag' => trim($object['ETag'], '"'),
+ 'size' => (int)($object['Size'] ?? $object['ContentLength']),
+ ];
+ }
+
+ public function listObjects(string $prefix = ''): \Iterator {
+ $results = $this->getConnection()->getPaginator('ListObjectsV2', [
+ 'Bucket' => $this->bucket,
+ 'Prefix' => $prefix,
+ ] + $this->getSSECParameters());
+
+ foreach ($results as $result) {
+ if (is_array($result['Contents'])) {
+ foreach ($result['Contents'] as $object) {
+ yield [
+ 'urn' => basename($object['Key']),
+ 'metadata' => [
+ 'mtime' => $object['LastModified'],
+ 'etag' => trim($object['ETag'], '"'),
+ 'size' => (int)($object['Size'] ?? $object['ContentLength']),
+ ],
+ ];
+ }
+ }
+ }
+ }
}
diff --git a/lib/public/Files/ObjectStore/IObjectStoreMetaData.php b/lib/public/Files/ObjectStore/IObjectStoreMetaData.php
new file mode 100644
index 0000000000000..8359e83f573ea
--- /dev/null
+++ b/lib/public/Files/ObjectStore/IObjectStoreMetaData.php
@@ -0,0 +1,38 @@
+
+ *
+ * @since 32.0.0
+ */
+ public function listObjects(string $prefix = ''): \Iterator;
+}