diff --git a/apps/files/appinfo/info.xml b/apps/files/appinfo/info.xml index cdcc79e47fb76..95f6153e1d710 100644 --- a/apps/files/appinfo/info.xml +++ b/apps/files/appinfo/info.xml @@ -49,6 +49,9 @@ OCA\Files\Command\Object\Delete OCA\Files\Command\Object\Get OCA\Files\Command\Object\Put + OCA\Files\Command\Object\Info + OCA\Files\Command\Object\ListObject + OCA\Files\Command\Object\Orphans diff --git a/apps/files/composer/composer/autoload_classmap.php b/apps/files/composer/composer/autoload_classmap.php index 0d9e6aa2d77dd..a74df7ba3d24f 100644 --- a/apps/files/composer/composer/autoload_classmap.php +++ b/apps/files/composer/composer/autoload_classmap.php @@ -35,7 +35,10 @@ 'OCA\\Files\\Command\\Move' => $baseDir . '/../lib/Command/Move.php', 'OCA\\Files\\Command\\Object\\Delete' => $baseDir . '/../lib/Command/Object/Delete.php', 'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php', + 'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php', + 'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php', 'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php', + 'OCA\\Files\\Command\\Object\\Orphans' => $baseDir . '/../lib/Command/Object/Orphans.php', 'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php', 'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php', 'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php', diff --git a/apps/files/composer/composer/autoload_static.php b/apps/files/composer/composer/autoload_static.php index 5ece90731783b..1d79f38e35a9a 100644 --- a/apps/files/composer/composer/autoload_static.php +++ b/apps/files/composer/composer/autoload_static.php @@ -50,7 +50,10 @@ class ComposerStaticInitFiles 'OCA\\Files\\Command\\Move' => __DIR__ . '/..' . '/../lib/Command/Move.php', 'OCA\\Files\\Command\\Object\\Delete' => __DIR__ . '/..' . '/../lib/Command/Object/Delete.php', 'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php', + 'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php', + 'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php', 'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php', + 'OCA\\Files\\Command\\Object\\Orphans' => __DIR__ . '/..' . '/../lib/Command/Object/Orphans.php', 'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php', 'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php', 'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php', diff --git a/apps/files/lib/Command/Object/Info.php b/apps/files/lib/Command/Object/Info.php new file mode 100644 index 0000000000000..6748de37cfe2b --- /dev/null +++ b/apps/files/lib/Command/Object/Info.php @@ -0,0 +1,80 @@ +setName('files:object:info') + ->setDescription('Get the metadata of an object') + ->addArgument('object', InputArgument::REQUIRED, 'Object to get') + ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to get the object from, only required in cases where it can't be determined from the config"); + } + + public function execute(InputInterface $input, OutputInterface $output): int { + $object = $input->getArgument('object'); + $objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output); + if (!$objectStore) { + return self::FAILURE; + } + + if (!$objectStore instanceof IObjectStoreMetaData) { + $output->writeln('Configured object store does currently not support retrieve metadata'); + return self::FAILURE; + } + + if (!$objectStore->objectExists($object)) { + $output->writeln("Object $object does not exist"); + return self::FAILURE; + } + + try { + $meta = $objectStore->getObjectMetaData($object); + } catch (\Exception $e) { + $msg = $e->getMessage(); + $output->writeln("Failed to read $object from object store: $msg"); + return self::FAILURE; + } + + if ($input->getOption('output') === 'plain' && isset($meta['size'])) { + $meta['size'] = Util::humanFileSize($meta['size']); + } + if (isset($meta['mtime'])) { + $meta['mtime'] = $meta['mtime']->format(\DateTimeImmutable::ATOM); + } + if (!isset($meta['mimetype'])) { + $handle = $objectStore->readObject($object); + $head = fread($handle, 8192); + fclose($handle); + $meta['mimetype'] = $this->mimeTypeDetector->detectString($head); + } + + $this->writeArrayInOutputFormat($input, $output, $meta); + + return self::SUCCESS; + } + +} diff --git a/apps/files/lib/Command/Object/ListObject.php b/apps/files/lib/Command/Object/ListObject.php new file mode 100644 index 0000000000000..5d30232e09f8f --- /dev/null +++ b/apps/files/lib/Command/Object/ListObject.php @@ -0,0 +1,50 @@ +setName('files:object:list') + ->setDescription('List all objects in the object store') + ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config"); + } + + public function execute(InputInterface $input, OutputInterface $output): int { + $objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output); + if (!$objectStore) { + return self::FAILURE; + } + + if (!$objectStore instanceof IObjectStoreMetaData) { + $output->writeln('Configured object store does currently not support listing objects'); + return self::FAILURE; + } + $objects = $objectStore->listObjects(); + $objects = $this->objectUtils->formatObjects($objects, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN); + $this->writeStreamingTableInOutputFormat($input, $output, $objects, self::CHUNK_SIZE); + + return self::SUCCESS; + } +} diff --git a/apps/files/lib/Command/Object/ObjectUtil.php b/apps/files/lib/Command/Object/ObjectUtil.php index c4ab59608fbac..5f053c2c42fff 100644 --- a/apps/files/lib/Command/Object/ObjectUtil.php +++ b/apps/files/lib/Command/Object/ObjectUtil.php @@ -12,6 +12,7 @@ use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; use OCP\IDBConnection; +use OCP\Util; use Symfony\Component\Console\Output\OutputInterface; class ObjectUtil { @@ -91,4 +92,24 @@ public function objectExistsInDb(string $object): int|false { return $fileId; } + + public function formatObjects(\Iterator $objects, bool $humanOutput): \Iterator { + foreach ($objects as $object) { + yield $this->formatObject($object, $humanOutput); + } + } + + public function formatObject(array $object, bool $humanOutput): array { + $row = array_merge([ + 'urn' => $object['urn'], + ], ($object['metadata'] ?? [])); + + if ($humanOutput && isset($row['size'])) { + $row['size'] = Util::humanFileSize($row['size']); + } + if (isset($row['mtime'])) { + $row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM); + } + return $row; + } } diff --git a/apps/files/lib/Command/Object/Orphans.php b/apps/files/lib/Command/Object/Orphans.php new file mode 100644 index 0000000000000..f7132540fc82f --- /dev/null +++ b/apps/files/lib/Command/Object/Orphans.php @@ -0,0 +1,79 @@ +query) { + $this->query = $this->connection->getQueryBuilder(); + $this->query->select('fileid') + ->from('filecache') + ->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id'))); + } + return $this->query; + } + + protected function configure(): void { + parent::configure(); + $this + ->setName('files:object:orphans') + ->setDescription('List all objects in the object store that don\'t have a matching entry in the database') + ->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config"); + } + + public function execute(InputInterface $input, OutputInterface $output): int { + $objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output); + if (!$objectStore) { + return self::FAILURE; + } + + if (!$objectStore instanceof IObjectStoreMetaData) { + $output->writeln('Configured object store does currently not support listing objects'); + return self::FAILURE; + } + $prefixLength = strlen('urn:oid:'); + + $objects = $objectStore->listObjects('urn:oid:'); + $orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) { + $fileId = (int)substr($object['urn'], $prefixLength); + return !$this->fileIdInDb($fileId); + }); + + $orphans = $this->objectUtils->formatObjects($orphans, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN); + $this->writeStreamingTableInOutputFormat($input, $output, $orphans, self::CHUNK_SIZE); + + return self::SUCCESS; + } + + private function fileIdInDb(int $fileId): bool { + $query = $this->getQuery(); + $query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT); + $result = $query->executeQuery(); + return $result->fetchOne() !== false; + } +} diff --git a/core/Command/Base.php b/core/Command/Base.php index b915ae2ae4a3a..c9b6337b64a9a 100644 --- a/core/Command/Base.php +++ b/core/Command/Base.php @@ -88,6 +88,58 @@ protected function writeTableInOutputFormat(InputInterface $input, OutputInterfa } } + protected function writeStreamingTableInOutputFormat(InputInterface $input, OutputInterface $output, \Iterator $items, int $tableGroupSize): void { + switch ($input->getOption('output')) { + case self::OUTPUT_FORMAT_JSON: + case self::OUTPUT_FORMAT_JSON_PRETTY: + $this->writeStreamingJsonArray($input, $output, $items); + break; + default: + foreach ($this->chunkIterator($items, $tableGroupSize) as $chunk) { + $this->writeTableInOutputFormat($input, $output, $chunk); + } + break; + } + } + + protected function writeStreamingJsonArray(InputInterface $input, OutputInterface $output, \Iterator $items): void { + $first = true; + $outputType = $input->getOption('output'); + + $output->writeln('['); + foreach ($items as $item) { + if (!$first) { + $output->writeln(','); + } + if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) { + $output->write(json_encode($item, JSON_PRETTY_PRINT)); + } else { + $output->write(json_encode($item)); + } + $first = false; + } + $output->writeln("\n]"); + } + + public function chunkIterator(\Iterator $iterator, int $count): \Iterator { + $chunk = []; + + for ($i = 0; $iterator->valid(); $i++) { + $chunk[] = $iterator->current(); + $iterator->next(); + if (count($chunk) == $count) { + // Got a full chunk, yield and start a new one + yield $chunk; + $chunk = []; + } + } + + if (count($chunk)) { + // Yield the last chunk even if incomplete + yield $chunk; + } + } + /** * @param mixed $item diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 2d0b62d749f7f..52e3075e4134b 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -456,6 +456,7 @@ 'OCP\\Files\\Notify\\INotifyHandler' => $baseDir . '/lib/public/Files/Notify/INotifyHandler.php', 'OCP\\Files\\Notify\\IRenameChange' => $baseDir . '/lib/public/Files/Notify/IRenameChange.php', 'OCP\\Files\\ObjectStore\\IObjectStore' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStore.php', + 'OCP\\Files\\ObjectStore\\IObjectStoreMetaData' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMetaData.php', 'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => $baseDir . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php', 'OCP\\Files\\ReservedWordException' => $baseDir . '/lib/public/Files/ReservedWordException.php', 'OCP\\Files\\Search\\ISearchBinaryOperator' => $baseDir . '/lib/public/Files/Search/ISearchBinaryOperator.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 64210bd584344..e98bc3e1aaaee 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -505,6 +505,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OCP\\Files\\Notify\\INotifyHandler' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/INotifyHandler.php', 'OCP\\Files\\Notify\\IRenameChange' => __DIR__ . '/../../..' . '/lib/public/Files/Notify/IRenameChange.php', 'OCP\\Files\\ObjectStore\\IObjectStore' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStore.php', + 'OCP\\Files\\ObjectStore\\IObjectStoreMetaData' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMetaData.php', 'OCP\\Files\\ObjectStore\\IObjectStoreMultiPartUpload' => __DIR__ . '/../../..' . '/lib/public/Files/ObjectStore/IObjectStoreMultiPartUpload.php', 'OCP\\Files\\ReservedWordException' => __DIR__ . '/../../..' . '/lib/public/Files/ReservedWordException.php', 'OCP\\Files\\Search\\ISearchBinaryOperator' => __DIR__ . '/../../..' . '/lib/public/Files/Search/ISearchBinaryOperator.php', diff --git a/lib/private/Files/ObjectStore/S3.php b/lib/private/Files/ObjectStore/S3.php index 41ab75caf45fe..e970fb6ac145a 100644 --- a/lib/private/Files/ObjectStore/S3.php +++ b/lib/private/Files/ObjectStore/S3.php @@ -3,14 +3,16 @@ * SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors * SPDX-License-Identifier: AGPL-3.0-or-later */ + namespace OC\Files\ObjectStore; use Aws\Result; use Exception; use OCP\Files\ObjectStore\IObjectStore; +use OCP\Files\ObjectStore\IObjectStoreMetaData; use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload; -class S3 implements IObjectStore, IObjectStoreMultiPartUpload { +class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaData { use S3ConnectionTrait; use S3ObjectTrait; @@ -61,7 +63,7 @@ public function getMultipartUploads(string $urn, string $uploadId): array { 'Key' => $urn, 'UploadId' => $uploadId, 'MaxParts' => 1000, - 'PartNumberMarker' => $partNumberMarker + 'PartNumberMarker' => $partNumberMarker, ] + $this->getSSECParameters()); $parts = array_merge($parts, $result->get('Parts') ?? []); $isTruncated = $result->get('IsTruncated'); @@ -89,7 +91,41 @@ public function abortMultipartUpload($urn, $uploadId): void { $this->getConnection()->abortMultipartUpload([ 'Bucket' => $this->bucket, 'Key' => $urn, - 'UploadId' => $uploadId + 'UploadId' => $uploadId, ]); } + + public function getObjectMetaData(string $urn): array { + $object = $this->getConnection()->headObject([ + 'Bucket' => $this->bucket, + 'Key' => $urn + ] + $this->getSSECParameters())->toArray(); + return [ + 'mtime' => $object['LastModified'], + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ]; + } + + public function listObjects(string $prefix = ''): \Iterator { + $results = $this->getConnection()->getPaginator('ListObjectsV2', [ + 'Bucket' => $this->bucket, + 'Prefix' => $prefix, + ] + $this->getSSECParameters()); + + foreach ($results as $result) { + if (is_array($result['Contents'])) { + foreach ($result['Contents'] as $object) { + yield [ + 'urn' => basename($object['Key']), + 'metadata' => [ + 'mtime' => $object['LastModified'], + 'etag' => trim($object['ETag'], '"'), + 'size' => (int)($object['Size'] ?? $object['ContentLength']), + ], + ]; + } + } + } + } } diff --git a/lib/public/Files/ObjectStore/IObjectStoreMetaData.php b/lib/public/Files/ObjectStore/IObjectStoreMetaData.php new file mode 100644 index 0000000000000..8359e83f573ea --- /dev/null +++ b/lib/public/Files/ObjectStore/IObjectStoreMetaData.php @@ -0,0 +1,38 @@ + + * + * @since 32.0.0 + */ + public function listObjects(string $prefix = ''): \Iterator; +}