feat: rework object listing

Signed-off-by: Robin Appelman <[email protected]>

Author: icewind1991

apps/files/lib/Command/Object/Info.php

@@ -11,6 +11,7 @@
 use OC\Core\Command\Base;
 use OCP\Files\IMimeTypeDetector;
 use OCP\Files\ObjectStore\IObjectStoreMetaData;
+use OCP\Util;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
@@ -59,7 +60,7 @@ public function execute(InputInterface $input, OutputInterface $output): int {
 		}
 
 		if ($input->getOption('output') === 'plain' && isset($meta['size'])) {
-			$meta['size'] = \OC_Helper::humanFileSize($meta['size']);
+			$meta['size'] = Util::humanFileSize($meta['size']);
 		}
 		if (isset($meta['mtime'])) {
 			$meta['mtime'] = $meta['mtime']->format(\DateTimeImmutable::ATOM);

apps/files/lib/Command/Object/ObjectUtil.php

@@ -13,6 +13,7 @@
 use OCP\Files\ObjectStore\IObjectStore;
 use OCP\IConfig;
 use OCP\IDBConnection;
+use OCP\Util;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 
@@ -97,32 +98,28 @@ public function objectExistsInDb(string $object): int|false {
 	public function writeIteratorToOutput(InputInterface $input, OutputInterface $output, \Iterator $objects, int $chunkSize): void {
 		$outputType = $input->getOption('output');
 		$humanOutput = $outputType === Base::OUTPUT_FORMAT_PLAIN;
-		$first = true;
 
-		if (!$humanOutput) {
-			$output->writeln('[');
-		}
+		if ($humanOutput) {
+			// we can't write tables in a streaming way, so we print them in chunks instead
+			foreach ($this->chunkIterator($objects, $chunkSize) as $chunk) {
+				$this->outputChunkHuman($input, $output, $chunk);
+			}
+		} else {
+			$first = true;
 
-		foreach ($this->chunkIterator($objects, $chunkSize) as $chunk) {
-			if ($outputType === Base::OUTPUT_FORMAT_PLAIN) {
-				$this->outputChunk($input, $output, $chunk);
-			} else {
-				foreach ($chunk as $object) {
-					if (!$first) {
-						$output->writeln(',');
-					}
-					$row = $this->formatObject($object, $humanOutput);
-					if ($outputType === Base::OUTPUT_FORMAT_JSON_PRETTY) {
-						$output->write(json_encode($row, JSON_PRETTY_PRINT));
-					} else {
-						$output->write(json_encode($row));
-					}
-					$first = false;
+			$output->writeln('[');
+			foreach ($objects as $object) {
+				if (!$first) {
+					$output->writeln(',');
 				}
+				$row = $this->formatObject($object, false);
+				if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) {
+					$output->write(json_encode($row, JSON_PRETTY_PRINT));
+				} else {
+					$output->write(json_encode($row));
+				}
+				$first = false;
 			}
-		}
-
-		if (!$humanOutput) {
 			$output->writeln("\n]");
 		}
 	}
@@ -133,20 +130,18 @@ private function formatObject(array $object, bool $humanOutput): array {
 		], ($object['metadata'] ?? []));
 
 		if ($humanOutput && isset($row['size'])) {
-			$row['size'] = \OC_Helper::humanFileSize($row['size']);
+			$row['size'] = Util::humanFileSize($row['size']);
 		}
 		if (isset($row['mtime'])) {
 			$row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM);
 		}
 		return $row;
 	}
 
-	private function outputChunk(InputInterface $input, OutputInterface $output, iterable $chunk): void {
+	private function outputChunkHuman(InputInterface $input, OutputInterface $output, iterable $chunk): void {
 		$result = [];
-		$humanOutput = $input->getOption('output') === 'plain';
-
 		foreach ($chunk as $object) {
-			$result[] = $this->formatObject($object, $humanOutput);
+			$result[] = $this->formatObject($object, true);
 		}
 		$this->writeTableInOutputFormat($input, $output, $result);
 	}
@@ -158,12 +153,14 @@ public function chunkIterator(\Iterator $iterator, int $count): \Iterator {
 			$chunk[] = $iterator->current();
 			$iterator->next();
 			if (count($chunk) == $count) {
+				// Got a full chunk, yield and start a new one
 				yield $chunk;
 				$chunk = [];
 			}
 		}
 
 		if (count($chunk)) {
+			// Yield the last chunk even if incomplete
 			yield $chunk;
 		}
 	}

apps/files/lib/Command/Object/Orphans.php

@@ -19,18 +19,23 @@
 class Orphans extends Base {
 	private const CHUNK_SIZE = 100;
 
-	private IQueryBuilder $query;
+	private ?IQueryBuilder $query = null;
 
 	public function __construct(
 		private readonly ObjectUtil $objectUtils,
-		IDBConnection $connection,
+		private readonly IDBConnection $connection,
 	) {
 		parent::__construct();
+	}
 
-		$this->query = $connection->getQueryBuilder();
-		$this->query->select('fileid')
-			->from('filecache')
-			->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id')));
+	private function getQuery(): IQueryBuilder {
+		if (!$this->query) {
+			$this->query = $this->connection->getQueryBuilder();
+			$this->query->select('fileid')
+				->from('filecache')
+				->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id')));
+		}
+		return $this->query;
 	}
 
 	protected function configure(): void {
@@ -54,20 +59,21 @@ public function execute(InputInterface $input, OutputInterface $output): int {
 		$prefixLength = strlen('urn:oid:');
 
 		$objects = $objectStore->listObjects('urn:oid:');
-		$objects->rewind();
 		$orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) {
 			$fileId = (int)substr($object['urn'], $prefixLength);
 			return !$this->fileIdInDb($fileId);
 		});
-		$orphans = new \ArrayIterator(iterator_to_array($orphans));
+		$orphans->rewind();
+
 		$this->objectUtils->writeIteratorToOutput($input, $output, $orphans, self::CHUNK_SIZE);
 
 		return self::SUCCESS;
 	}
 
 	private function fileIdInDb(int $fileId): bool {
-		$this->query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT);
-		$result = $this->query->executeQuery();
+		$query = $this->getQuery();
+		$query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT);
+		$result = $query->executeQuery();
 		return $result->fetchOne() !== false;
 	}
 }