diff --git a/io/io/src/TDirectoryFile.cxx b/io/io/src/TDirectoryFile.cxx index d4f33a9c12a6e..34ae71942a592 100644 --- a/io/io/src/TDirectoryFile.cxx +++ b/io/io/src/TDirectoryFile.cxx @@ -1258,7 +1258,9 @@ TFile *TDirectoryFile::OpenFile(const char *name, Option_t *option,const char *f /// Create a sub-directory "a" or a hierarchy of sub-directories "a/b/c/...". /// /// @param name the name or hierarchy of the subdirectory ("a" or "a/b/c") -/// @param title the title +/// @param title the title of the directory. For hierarchies, this is only applied +/// to the innermost directory (so if `name == "a/b/c"` and `title == "my dir"`, +/// only `c` will have the title `"my dir"`). /// @param returnExistingDirectory if key-name is already existing, the returned /// value points to preexisting sub-directory if true and to `nullptr` if false. /// @return a pointer to the created sub-directory, not to the top sub-directory @@ -1284,10 +1286,11 @@ TDirectory *TDirectoryFile::mkdir(const char *name, const char *title, Bool_t re TDirectoryFile *tmpdir = nullptr; GetObject(workname.Data(), tmpdir); if (!tmpdir) { - tmpdir = (TDirectoryFile*)mkdir(workname.Data(),title); + // We give all intermediate directories a default title, as `title` is only given to the innermost dir. + tmpdir = (TDirectoryFile *)mkdir(workname.Data(), workname.Data()); if (!tmpdir) return nullptr; } - return tmpdir->mkdir(slash + 1); + return tmpdir->mkdir(slash + 1, title, returnExistingDirectory); } TDirectory::TContext ctxt(this); diff --git a/io/io/test/TFileTests.cxx b/io/io/test/TFileTests.cxx index 16c58e8fd46a6..e33ddfc59ec66 100644 --- a/io/io/test/TFileTests.cxx +++ b/io/io/test/TFileTests.cxx @@ -5,6 +5,8 @@ #include "gtest/gtest.h" +#include + #include "TFile.h" #include "TMemFile.h" #include "TDirectory.h" @@ -280,6 +282,28 @@ TEST(TDirectoryFile, SeekParent) EXPECT_EQ(dir11->GetSeekParent(), 239); } +TEST(TDirectoryFile, RecursiveMkdir) +{ + TMemFile f("mkdirtest.root", "RECREATE"); + auto dir1 = f.mkdir("a/b/c", "my dir"); + EXPECT_NE(dir1, nullptr); + { + ROOT::TestSupport::CheckDiagsRAII diags; + diags.requiredDiag(kError, "TDirectoryFile::mkdir","An object with name c exists already"); + auto dir2 = f.mkdir("a/b/c", "", /* returnExisting = */ false); + EXPECT_EQ(dir2, nullptr); + } + auto dir3 = f.mkdir("a/b/c", "foobar", /* returnExisting = */ true); + EXPECT_EQ(dir3, dir1); + EXPECT_STREQ(dir3->GetTitle(), "my dir"); + auto dirB = dir3->GetMotherDir(); + ASSERT_NE(dirB, nullptr); + EXPECT_STREQ(dirB->GetTitle(), "b"); + auto dirA = dirB->GetMotherDir(); + ASSERT_NE(dirA, nullptr); + EXPECT_STREQ(dirA->GetTitle(), "a"); +} + // https://its.cern.ch/jira/browse/ROOT-10581 TEST(TFile, PersistTObjectStdArray) { diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 98e50fd87e853..a1a186faedbd5 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -136,6 +136,7 @@ add_dependencies(rootcint rootcint) if (TARGET Gui) ROOT_EXECUTABLE(rootbrowse src/rootbrowse.cxx LIBRARIES RIO Core Rint Gui) endif() +ROOT_EXECUTABLE(rootcp src/rootcp.cxx LIBRARIES RIO Tree Core Rint) ROOT_EXECUTABLE(rootls src/rootls.cxx LIBRARIES RIO Tree Core Rint ROOTNTuple) ROOT_ADD_TEST_SUBDIRECTORY(test) diff --git a/main/python/cmdLineUtils.py b/main/python/cmdLineUtils.py index f3af283b197db..15705f68f1268 100644 --- a/main/python/cmdLineUtils.py +++ b/main/python/cmdLineUtils.py @@ -783,52 +783,6 @@ def deleteRootObject(rootFile, pathSplit, interactive, recursive): # End of help strings ########## -########## -# ROOTCP - - -def _copyObjects(fileName, pathSplitList, destFile, destPathSplit, oneFile, recursive, replace): - retcode = 0 - destFileName = destFile.GetName() - rootFile = openROOTFile(fileName) if fileName != destFileName else destFile - if not rootFile: - return 1 - ROOT.gROOT.GetListOfFiles().Remove(rootFile) # Fast copy necessity - for pathSplit in pathSplitList: - oneSource = oneFile and len(pathSplitList) == 1 - retcode += copyRootObject(rootFile, pathSplit, destFile, destPathSplit, oneSource, recursive, replace) - if fileName != destFileName: - rootFile.Close() - return retcode - - -def rootCp(sourceList, destFileName, destPathSplit, compress=None, recreate=False, recursive=False, replace=False): - # Check arguments - if sourceList == [] or destFileName == "": - return 1 - if recreate and destFileName in [n[0] for n in sourceList]: - logging.error("cannot recreate destination file if this is also a source file") - return 1 - - # Open destination file - destFile = openROOTFileCompress(destFileName, compress, recreate) - if not destFile: - return 1 - ROOT.gROOT.GetListOfFiles().Remove(destFile) # Fast copy necessity - - # Loop on the root files - retcode = 0 - for fileName, pathSplitList in sourceList: - retcode += _copyObjects( - fileName, pathSplitList, destFile, destPathSplit, len(sourceList) == 1, recursive, replace - ) - destFile.Close() - return retcode - - -# End of ROOTCP -########## - ########## # ROOTEVENTSELECTOR diff --git a/main/python/rootcp.py b/main/python/rootcp.py deleted file mode 100755 index aada781718ece..0000000000000 --- a/main/python/rootcp.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env @python@ - -# ROOT command line tools: rootcp -# Author: Julien Ripoche -# Mail: julien.ripoche@u-psud.fr -# Date: 20/08/15 - -"""Command line to copy objects from ROOT files into an other""" - -import cmdLineUtils -import sys - - -# Help strings -description = "Copy objects from ROOT files into an other" - -EPILOG = """ -Note: If an object has been written to a file multiple times, rootcp will copy only the latest version of that object. - -Examples: -- rootcp source.root dest.root - Copy the latest version of each object in 'source.root' to 'dest.root'. - -- rootcp source.root:hist* dest.root - Copy all histograms whose names start with 'hist' from 'source.root' to 'dest.root'. - -- rootcp source1.root:hist1 source2.root:hist2 dest.root - Copy histograms 'hist1' from 'source1.root' and 'hist2' from 'source2.root' to 'dest.root'. - -- rootcp --recreate source.root:hist dest.root - Recreate 'dest.root' and copy the histogram named 'hist' from 'source.root' into it. - -- rootcp -c 101 source.root:hist dest.root - Change compression, if not existing, of 'dest.root' to ZLIB algorithm with compression level 1 and copy the histogram named 'hist' from 'source.root' into it. - Meaning of the '-c' argument is given by 'compress = 100 * algorithm + level'. - Other examples of usage: - * -c 509 : ZSTD with compression level 9 - * -c 404 : LZ4 with compression level 4 - * -c 207 : LZMA with compression level 7 - For more information see https://root.cern.ch/doc/master/classTFile.html#ad0377adf2f3d88da1a1f77256a140d60 - and https://root.cern.ch/doc/master/structROOT_1_1RCompressionSetting.html - - """ - -def get_argparse(): - # Collect arguments with the module argparse - parser = cmdLineUtils.getParserSourceDest(description, EPILOG) - parser.prog = 'rootcp' - parser.add_argument("-c","--compress", type=int, help=cmdLineUtils.COMPRESS_HELP) - parser.add_argument("--recreate", help=cmdLineUtils.RECREATE_HELP, action="store_true") - parser.add_argument("-r","--recursive", help=cmdLineUtils.RECURSIVE_HELP, action="store_true") - parser.add_argument("--replace", help=cmdLineUtils.REPLACE_HELP, action="store_true") - return parser - - -def execute(): - parser = get_argparse() - # Put arguments in shape - sourceList, destFileName, destPathSplit, optDict = cmdLineUtils.getSourceDestListOptDict(parser) - - # Process rootCp - return cmdLineUtils.rootCp(sourceList, destFileName, destPathSplit, \ - compress=optDict["compress"], recreate=optDict["recreate"], \ - recursive=optDict["recursive"], replace=optDict["replace"]) -if __name__ == "__main__": - sys.exit(execute()) diff --git a/main/src/RootObjTree.cxx b/main/src/RootObjTree.cxx new file mode 100644 index 0000000000000..3f994fb8c7a1b --- /dev/null +++ b/main/src/RootObjTree.cxx @@ -0,0 +1,234 @@ +// \file RootObjTree.cxx +/// +/// \author Giacomo Parolini +/// \date 2025-10-14 + +#include "RootObjTree.hxx" + +#include "wildcards.hpp" + +#include + +#include + +#include +#include +#include +#include + +static bool MatchesGlob(std::string_view haystack, std::string_view pattern) +{ + return wildcards::match(haystack, pattern); +} + +ROOT::CmdLine::RootSource +ROOT::CmdLine::GetMatchingPathsInFile(std::string_view fileName, std::string_view pattern, std::uint32_t flags) +{ + ROOT::CmdLine::RootSource source; + source.fFileName = fileName; + auto &nodeTree = source.fObjectTree; + nodeTree.fFile = std::unique_ptr(TFile::Open(std::string(fileName).c_str(), "READ")); + if (!nodeTree.fFile) { + source.fErrors.push_back("Failed to open file"); + return source; + } + + const auto patternSplits = pattern.empty() ? std::vector{} : ROOT::Split(pattern, "/"); + + // Match all objects at all nesting levels down to the deepest nesting level of `pattern` (or all nesting levels + // if we have the "recursive listing" flag). The nodes are visited breadth-first. + { + ROOT::CmdLine::RootObjNode rootNode = {}; + rootNode.fName = std::string(fileName); + rootNode.fClassName = nodeTree.fFile->Class()->GetName(); + rootNode.fDir = nodeTree.fFile.get(); + nodeTree.fNodes.emplace_back(std::move(rootNode)); + } + std::deque nodesToVisit{0}; + + // Keep track of the object names found at every nesting level and only add the first one. + std::unordered_set namesFound; + + const bool isRecursive = flags & EGetMatchingPathsFlags::kRecursive; + do { + NodeIdx_t curIdx = nodesToVisit.front(); + nodesToVisit.pop_front(); + ROOT::CmdLine::RootObjNode *cur = &nodeTree.fNodes[curIdx]; + assert(cur->fDir); + + // Sort the keys by name + std::vector keys; + keys.reserve(cur->fDir->GetListOfKeys()->GetEntries()); + for (TKey *key : ROOT::Detail::TRangeStaticCast(cur->fDir->GetListOfKeys())) + keys.push_back(key); + + std::sort(keys.begin(), keys.end(), + [](const auto *a, const auto *b) { return strcmp(a->GetName(), b->GetName()) < 0; }); + + namesFound.clear(); + + for (TKey *key : keys) { + // Don't recurse lower than requested by `pattern` unless we explicitly have the `recursive listing` flag. + if (cur->fNesting < patternSplits.size() && !MatchesGlob(key->GetName(), patternSplits[cur->fNesting])) + continue; + + if (namesFound.count(key->GetName()) > 0) { + std::cerr << "WARNING: Several versions of '" << key->GetName() << "' are present in '" << fileName + << "'. Only the most recent will be considered.\n"; + continue; + } + namesFound.insert(key->GetName()); + + auto &newChild = nodeTree.fNodes.emplace_back(NodeFromKey(*key)); + // Need to get back cur since the emplace_back() may have moved it. + cur = &nodeTree.fNodes[curIdx]; + newChild.fNesting = cur->fNesting + 1; + newChild.fParent = curIdx; + if (!cur->fNChildren) + cur->fFirstChild = nodeTree.fNodes.size() - 1; + cur->fNChildren++; + + const auto *cl = TClass::GetClass(key->GetClassName()); + if (cl && cl->InheritsFrom("TDirectory")) + newChild.fDir = cur->fDir->GetDirectory(key->GetName()); + } + + // Only recurse into subdirectories that are up to the deepest level we ask for through `pattern`. + if (cur->fNesting < patternSplits.size() || isRecursive) { + for (auto childIdx = cur->fFirstChild; childIdx < cur->fFirstChild + cur->fNChildren; ++childIdx) { + auto &child = nodeTree.fNodes[childIdx]; + if (child.fDir) + nodesToVisit.push_back(childIdx); + else if (cur->fNesting < patternSplits.size()) + nodeTree.fLeafList.push_back(childIdx); + } + } + if (cur->fNesting == patternSplits.size()) { + if (cur->fDir) + nodeTree.fDirList.push_back(curIdx); + else + nodeTree.fLeafList.push_back(curIdx); + } + } while (!nodesToVisit.empty()); + + return source; +} + +ROOT::RResult> +ROOT::CmdLine::SplitIntoFileNameAndPattern(std::string_view sourceRaw) +{ + auto prefixIdx = sourceRaw.find("://"); + std::string_view::size_type separatorIdx = 0; + if (prefixIdx != std::string_view::npos) { + bool prefixFound = false; + // Handle known URI prefixes + static const char *const specialPrefixes[] = {"http", "https", "root", "gs", "s3"}; + auto prefix = sourceRaw.substr(0, prefixIdx); + for (std::string_view knownPrefix : specialPrefixes) { + if (prefix == knownPrefix) { + prefixFound = true; + break; + } + } + if (!prefixFound) { + return R__FAIL("unknown file protocol"); + } + separatorIdx = sourceRaw.substr(prefixIdx + 3).find_first_of(':'); + if (separatorIdx != std::string_view::npos) + separatorIdx += prefixIdx + 3; + } else { + separatorIdx = sourceRaw.find_first_of(':'); + } + + if (separatorIdx != std::string_view::npos) { + return {{sourceRaw.substr(0, separatorIdx), sourceRaw.substr(separatorIdx + 1)}}; + } + return {{sourceRaw, std::string_view{}}}; +} + +ROOT::CmdLine::RootSource ROOT::CmdLine::ParseRootSource(std::string_view sourceRaw, std::uint32_t flags) +{ + ROOT::CmdLine::RootSource source; + + auto res = SplitIntoFileNameAndPattern(sourceRaw); + if (!res) { + source.fErrors.push_back(res.GetError()->GetReport()); + return source; + } + + auto [fileName, tokens] = res.Unwrap(); + source = ROOT::CmdLine::GetMatchingPathsInFile(fileName, tokens, flags); + + assert(source.fErrors.empty() == !!source.fObjectTree.fFile); + return source; +} + +std::vector +ROOT::CmdLine::ParseRootSources(const std::vector &sourcesRaw, std::uint32_t flags) +{ + std::vector sources; + sources.reserve(sourcesRaw.size()); + + for (const auto &srcRaw : sourcesRaw) { + sources.push_back(ParseRootSource(srcRaw, flags)); + } + + return sources; +} + +void ROOT::CmdLine::PrintObjTree(const RootObjTree &tree, std::ostream &out) +{ + if (tree.fNodes.empty()) + return; + + struct RevNode { + std::set fChildren; + }; + std::vector revNodes; + revNodes.resize(tree.fNodes.size()); + + // Un-linearize the tree + for (int i = (int)tree.fNodes.size() - 1; i >= 0; --i) { + const auto *node = &tree.fNodes[i]; + NodeIdx_t childIdx = i; + NodeIdx_t parentIdx = node->fParent; + while (childIdx != parentIdx) { + auto &revNodeParent = revNodes[parentIdx]; + revNodeParent.fChildren.insert(childIdx); + node = &tree.fNodes[parentIdx]; + childIdx = parentIdx; + parentIdx = node->fParent; + } + } + + // Print out the tree. + // Vector of {nesting, nodeIdx} + std::vector> nodesToVisit = {{0, 0}}; + while (!nodesToVisit.empty()) { + const auto [nesting, nodeIdx] = nodesToVisit.back(); + nodesToVisit.pop_back(); + const auto &cur = revNodes[nodeIdx]; + const auto &node = tree.fNodes[nodeIdx]; + for (auto i = 0u; i < 2 * nesting; ++i) + out << ' '; + out << node.fName << " : " << node.fClassName << "\n"; + // Add the children in reverse order to preserve alphabetical order during depth-first visit. + for (auto it = cur.fChildren.rbegin(); it != cur.fChildren.rend(); ++it) { + nodesToVisit.push_back({nesting + 1, *it}); + } + } +} + +std::string ROOT::CmdLine::NodeFullPath(const ROOT::CmdLine::RootObjTree &tree, ROOT::CmdLine::NodeIdx_t nodeIdx, + ROOT::CmdLine::ENodeFullPathOpt opt) +{ + const RootObjNode *node = &tree.fNodes[nodeIdx]; + std::string fullPath = node->fName; + while (node->fParent != 0) { + node = &tree.fNodes[node->fParent]; + fullPath = node->fName + (fullPath.empty() ? "" : "/") + fullPath; + } + if (opt == ENodeFullPathOpt::kIncludeFilename && nodeIdx > 0) + fullPath = tree.fNodes[0].fName + ":" + fullPath; + return fullPath; +} diff --git a/main/src/RootObjTree.hxx b/main/src/RootObjTree.hxx new file mode 100644 index 0000000000000..0fd123dcaa8ca --- /dev/null +++ b/main/src/RootObjTree.hxx @@ -0,0 +1,120 @@ +// \file RootObjTree.hxx +/// +/// Utility functions used by command line tools to parse "path-like" strings like: "foo.root:dir/obj*" into a +/// tree structure usable to iterate the matched objects. +/// +/// For example usage, see rootls.cxx +/// +/// \author Giacomo Parolini +/// \date 2025-10-14 + +#ifndef ROOT_CMDLINE_OBJTREE +#define ROOT_CMDLINE_OBJTREE + +#include +#include +#include +#include +#include + +#include + +#include + +class TDirectory; +class TFile; + +namespace ROOT::CmdLine { + +using NodeIdx_t = std::uint32_t; + +struct RootObjNode { + std::string fName; + std::string fClassName; + TKey *fKey = nullptr; // This is non-null for all nodes except the root node (which is the file itself) + + TDirectory *fDir = nullptr; // This is null for all non-directory nodes + // NOTE: by construction of the tree, all children of the same node are contiguous. + NodeIdx_t fFirstChild = 0; + std::uint32_t fNChildren = 0; + std::uint32_t fNesting = 0; + NodeIdx_t fParent = 0; +}; + +inline RootObjNode NodeFromKey(TKey &key) +{ + RootObjNode node = {}; + node.fName = key.GetName(); + node.fClassName = key.GetClassName(); + node.fKey = &key; + return node; +} + +struct RootObjTree { + // 0th node is the root node + std::vector fNodes; + // All nodes in fNodes that are dirs + std::vector fDirList; + // All nodes in fNodes that are leaves (non-TDirectories) + std::vector fLeafList; + // The file must be kept alive in order to access the nodes' keys + std::unique_ptr fFile; +}; + +/// Prints out the structure of the object tree. Helpful for debugging. +void PrintObjTree(const RootObjTree &tree, std::ostream &out = std::cout); + +enum class ENodeFullPathOpt { + kExcludeFilename, + kIncludeFilename, +}; +/// Given a node, returns its full path. If `opt == kIncludeFilename`, the path is prepended by "filename.root:" +std::string +NodeFullPath(const RootObjTree &tree, NodeIdx_t nodeIdx, ENodeFullPathOpt opt = ENodeFullPathOpt::kExcludeFilename); + +struct RootSource { + std::string fFileName; + RootObjTree fObjectTree; + std::vector fErrors; +}; + +enum EGetMatchingPathsFlags { + /// Recurse into subdirectories when matching objects + kRecursive = 1 << 0, +}; + +/// Given a file and a "path pattern", returns a RootSource containing the tree of matched objects. +/// +/// \param fileName The name of the ROOT file to look into +/// \param pattern A glob-like pattern (basically a `ls` pattern). May be empty to match anything. +/// \param flags A bitmask of EGetMatchingPathsFlags +RootSource GetMatchingPathsInFile(std::string_view fileName, std::string_view pattern, std::uint32_t flags); + +/// Given a string like "root://file.root:a/b/c", splits it into { "root://file.root", "a/b/c" }. +/// \return An error if the file prefix is unknown (e.g. "foo://file.root"), otherwise the result described above. +ROOT::RResult> SplitIntoFileNameAndPattern(std::string_view sourceRaw); + +/// Given a string like "file.root:dir/obj", converts it to a RootSource. +/// The string may start with one of the known file protocols: "http", "https", "root", "gs", "s3" +/// (e.g. "https://file.root"). +/// +/// If the source fails to get created, its fErrors list will be non-empty. +/// +/// \param flags A bitmask of EGetMatchingPathsFlags +/// \return The converted source. +RootSource ParseRootSource(std::string_view sourceRaw, std::uint32_t flags); + +/// Given a list of strings like "file.root:dir/obj", converts each string to a RootSource. +/// The string may start with one of the known file protocols: "http", "https", "root", "gs", "s3" +/// (e.g. "https://file.root"). +/// +/// If one or more sources fail to get created, each sources's fErrors list will be non-empty. +/// +/// \param flags A bitmask of EGetMatchingPathsFlags +/// \return The list of converted sources. +std::vector +ParseRootSources(const std::vector &sourcesRaw, std::uint32_t flags); + +} // namespace ROOT::CmdLine + +#endif diff --git a/main/src/rootcp.cxx b/main/src/rootcp.cxx new file mode 100644 index 0000000000000..48bc3a33bb78c --- /dev/null +++ b/main/src/rootcp.cxx @@ -0,0 +1,413 @@ +/// \file rootcp.cxx +/// +/// Command line tool to copy objects from ROOT files to others +/// +/// \author Giacomo Parolini +/// \date 2025-10-09 +#include + +#include "logging.hxx" +#include "optparse.hxx" +#include "RootObjTree.hxx" +#include "RootObjTree.cxx" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace ROOT::CmdLine; + +static const char *const kShortHelp = "usage: rootcp [-h] [-c COMPRESS] [--recreate] [-r|--recursive] [--replace] " + "[-v|--verbose] SOURCE [SOURCE ...] DEST\n"; +static const char *const kLongHelp = R"( +Copy objects from ROOT files into another + +positional arguments: + SOURCE Source file(s) + DEST Destination file + +options: + -h, --help show this help message and exit + -c, --compress COMPRESS + change the compression settings of the destination file (if not already + existing). + --recreate recreate the destination file. + -r, --recursive recurse inside directories + --replace replace object if already existing + -v be verbose + -vv be even more verbose + +Note: If an object has been written to a file multiple times, rootcp will copy only the latest version of that object. + +Source and destination files accept the syntax: `protocol://path/to/file.root:path/to/object*` to select specific +subobjects or directories in the file. + +Examples: +- rootcp source.root dest.root + Copy the latest version of each object in 'source.root' to 'dest.root'. + +- rootcp source.root:hist* dest.root + Copy all histograms whose names start with 'hist' from 'source.root' to 'dest.root'. + +- rootcp source1.root:hist1 source2.root:hist2 dest.root + Copy histograms 'hist1' from 'source1.root' and 'hist2' from 'source2.root' to 'dest.root'. + +- rootcp --recreate source.root:hist dest.root + Recreate 'dest.root' and copy the histogram named 'hist' from 'source.root' into it. + +- rootcp -c 101 source.root:hist dest.root + Change compression, if not existing, of 'dest.root' to ZLIB algorithm with compression level 1 and copy the histogram named 'hist' from 'source.root' into it. + Meaning of the '-c' argument is given by 'compress = 100 * algorithm + level'. + Other examples of usage: + * -c 509 : ZSTD with compression level 9 + * -c 404 : LZ4 with compression level 4 + * -c 207 : LZMA with compression level 7 + For more information see https://root.cern.ch/doc/latest-stable/classTFile.html#ad0377adf2f3d88da1a1f77256a140d60 + and https://root.cern.ch/doc/latest-stable/structROOT_1_1RCompressionSetting.html +)"; + +struct RootCpArgs { + enum class EPrintUsage { + kNo, + kShort, + kLong + }; + EPrintUsage fPrintHelp = EPrintUsage::kNo; + std::optional fCompression = std::nullopt; + bool fRecreate = false; + bool fReplace = false; + bool fRecursive = false; + std::vector fSources; +}; + +static RootCpArgs ParseArgs(const char **args, int nArgs) +{ + using ROOT::RCmdLineOpts; + + RootCpArgs outArgs; + + RCmdLineOpts opts; + opts.AddFlag({"-c", "--compress"}, RCmdLineOpts::EFlagType::kWithArg); + opts.AddFlag({"--recreate"}); + opts.AddFlag({"--replace"}); + opts.AddFlag({"-r", "--recursive"}); + opts.AddFlag({"-h", "--help"}); + opts.AddFlag({"-v"}); + opts.AddFlag({"-vv"}); + + opts.Parse(args, nArgs); + + for (const auto &err : opts.GetErrors()) { + std::cerr << err << "\n"; + } + if (!opts.GetErrors().empty()) { + outArgs.fPrintHelp = RootCpArgs::EPrintUsage::kShort; + return outArgs; + } + + if (opts.GetSwitch("help")) { + outArgs.fPrintHelp = RootCpArgs::EPrintUsage::kLong; + return outArgs; + } + + if (auto val = opts.GetFlagValueAs("compress"); val) + outArgs.fCompression = val; + outArgs.fRecursive = opts.GetSwitch("recursive"); + outArgs.fReplace = opts.GetSwitch("replace"); + outArgs.fRecreate = opts.GetSwitch("recreate"); + + if (opts.GetSwitch("vv")) + SetLogVerbosity(3); + else if (opts.GetSwitch("v")) + SetLogVerbosity(2); + + outArgs.fSources = opts.GetArgs(); + if (outArgs.fSources.size() < 2) + outArgs.fPrintHelp = RootCpArgs::EPrintUsage::kShort; + + return outArgs; +} + +static std::unique_ptr OpenFile(const char *fileName, const char *mode) +{ + const auto origLv = gErrorIgnoreLevel; + gErrorIgnoreLevel = kError; + auto file = std::unique_ptr(TFile::Open(fileName, mode)); + if (!file || file->IsZombie()) { + Err() << "File " << fileName << "does not exist.\n"; + return nullptr; + } + gErrorIgnoreLevel = origLv; + return file; +} + +namespace { + +struct RootCpDestination { + TFile *fFile; + std::string fPath; + std::string fFname; + bool fIsNewObject; +}; + +} // namespace + +// Splits `path` into a directory path (excluding the trailing '/') and a basename. +static std::pair DecomposePath(std::string_view path) +{ + auto lastSlashIdx = path.rfind('/'); + if (lastSlashIdx == std::string_view::npos) + return {{}, path}; + + auto dirName = path.substr(0, lastSlashIdx); + auto pathName = path.substr(lastSlashIdx + 1); + return {dirName, pathName}; +} + +static void CopyNode(const RootSource &src, const RootCpDestination &dest, NodeIdx_t nodeIdx, const RootCpArgs &args) +{ + TFile *srcfile = src.fObjectTree.fFile.get(); + const RootObjNode &node = src.fObjectTree.fNodes[nodeIdx]; + const std::string srcFullPath = NodeFullPath(src.fObjectTree, nodeIdx); + // Directory path, excluding trailing '/' and without the "file.root:" prefix. + const std::string_view srcDirPath = + (node.fParent == 0) ? std::string_view{} + : std::string_view{srcFullPath.data(), srcFullPath.size() - node.fName.size() - 1}; + + // Figure out where the output goes. If the user specified an output path (i.e. if dest.fPath is not empty), then + // use that. Otherwise, use the same path as the source object. + std::string destFullPath; + std::string_view destDirPath, destBaseName; + if (dest.fIsNewObject || dest.fPath.empty()) { + // User gave a destination which is not an existing directory or no destination at all + destFullPath = dest.fPath.empty() ? srcFullPath : dest.fPath; + auto decomposed = DecomposePath(destFullPath); + destDirPath = decomposed.first; + destBaseName = decomposed.second; + } else if (!dest.fPath.empty()) { + // User gave a destination which is an existing directory + destDirPath = dest.fPath; + destFullPath = std::string(destDirPath) + "/" + node.fName; + } + + Info(2) << "cp " << src.fFileName << ":" << srcFullPath << " -> " << dest.fFname << ":" << destFullPath << "\n"; + + TDirectory *destDir = dest.fFile; + if (!destDirPath.empty()) { + Info(3) << "mkdir " << destDirPath << "\n"; + destDir = dest.fFile->mkdir(std::string(destDirPath).c_str(), /* title = */ "", + /* returnPreExisting = */ true); + } + + // Check if the destination already exists. There are 3 cases here: + // 1. it doesn't: just go on as normal; + // 2. it does and it is a directory: the copied object needs to be copied inside it, but this was already accounted + // for outside CopyNode, so just go on as normal; + // 3. it does and it's not a directory: if we have the replace flag, replace it, otherwise error out. + const TKey *destKey = destDir->GetKey(std::string(destBaseName).c_str()); + if (destKey && !TClass::GetClass(destKey->GetClassName())->InheritsFrom("TDirectory") && !args.fReplace) { + Err() << "an object of type '" << destKey->GetClassName() << "' already exists at " << dest.fFname << ':' + << destFullPath << ". Use the --replace flag to overwrite existing objects.\n"; + return; + } + + // retrieve the object's key + const TDirectory *srcDir = srcfile->GetDirectory(std::string(srcDirPath).c_str(), true); + if (!srcDir) { + Err() << "failed to get source directory '" << srcDirPath << "'\n"; + return; + } + const TKey *srcKey = srcDir->GetKey(node.fName.c_str()); + if (!srcKey) { + Err() << "failed to read key of object '" << srcFullPath << "'\n"; + return; + } + + // Verify that the class is known and supported. + const std::string &className = node.fClassName; + const TClass *cl = TClass::GetClass(className.c_str()); + if (!cl) { + Err() << "unknown object type: " << className << "; object will be skipped.\n"; + return; + } + + Info(3) << "read object \"" << srcFullPath << "\" of type " << node.fClassName << "\n"; + if (!destDir) { + Err() << "failed to create or get destination directory \"" << dest.fFname << ":" << destDirPath << "\"\n"; + return; + } + + // Delete previous object if we're replacing it + if (destKey && args.fReplace) + destDir->Delete((std::string(destBaseName) + ";*").c_str()); + + // + // Do the actual copy + // + if (cl->InheritsFrom("TObject")) { + TObject *obj = node.fKey->ReadObj(); + if (!obj) { + Err() << "failed to read object \"" << srcFullPath << "\".\n"; + return; + } + + if (TTree *old = dynamic_cast(obj)) { + // special case for TTree + TDirectory::TContext ctx(gDirectory, destDir); + obj = old->CloneTree(-1, "fast"); + if (dest.fIsNewObject) { + static_cast(obj)->SetName(std::string(destBaseName).c_str()); + } + obj->Write(); + old->Delete(); + } else if (cl->InheritsFrom("TDirectory")) { + // directory + if (!args.fRecursive) { + Warn() << "Directory '" << srcFullPath + << "' will not be copied. Use the -r option if you need a recursive copy.\n"; + } else { + destDir->mkdir(node.fName.c_str(), srcKey->GetTitle(), true); + RootCpDestination dest2 = dest; + dest2.fPath = dest.fPath + (dest.fPath.empty() ? "" : "/") + node.fName; + for (auto childIdx = node.fFirstChild; childIdx < node.fFirstChild + node.fNChildren; ++childIdx) + CopyNode(src, dest2, childIdx, args); + } + } else { + // regular TObject + destDir->WriteObject(obj, std::string(destBaseName).c_str()); + } + obj->Delete(); + } else if (cl == TClass::GetClass("ROOT::RNTuple")) { + // TODO: RNTuple + Warn() << "object '" << node.fName << "' of type '" << node.fClassName + << "' will not be copied, as its type is currently unsupported by rootcp.\n"; + } else { + Warn() << "object '" << node.fName << "' of type '" << node.fClassName + << "' will not be copied, as its type is currently unsupported by rootcp.\n"; + } +} + +int main(int argc, char **argv) +{ + InitLog("rootcp"); + + // Parse arguments + auto args = ParseArgs(const_cast(argv) + 1, argc - 1); + if (args.fPrintHelp != RootCpArgs::EPrintUsage::kNo) { + std::cerr << kShortHelp; + if (args.fPrintHelp == RootCpArgs::EPrintUsage::kLong) { + std::cerr << kLongHelp; + return 0; + } + return 1; + } + + // Get destination. In general it may be a string like "prefix://file.root:path/to/dir", so check if it refers to + // a valid location. + // First validate the destination syntax. + const auto destFnameAndPattern = args.fSources.back(); + auto splitRes = SplitIntoFileNameAndPattern(destFnameAndPattern); + if (!splitRes) { + Err() << splitRes.GetError()->GetReport() << "\n"; + return 1; + } + auto [destFname, destPath] = splitRes.Unwrap(); + + // Check if the operation is allowed. + args.fSources.pop_back(); + if (args.fRecreate && std::find(args.fSources.begin(), args.fSources.end(), destFname) != args.fSources.end()) { + Err() << "cannot recreate destination file if this is also a source file\n"; + return 1; + } + + if (args.fCompression && gSystem->AccessPathName(std::string(destFname).c_str())) { + Err() << "can't change compression settings on existing file " << destFname << "\n"; + return 1; + } + + const char *destFileMode = + args.fRecreate ? "RECREATE_WITHOUT_GLOBALREGISTRATION" : "UPDATE_WITHOUT_GLOBALREGISTRATION"; + auto destFile = OpenFile(std::string(destFname).c_str(), destFileMode); + if (!destFile) + return 1; + + // `destPath` is the part after the colon (the input is given as `destFname:destPath`). It may be empty, but + // if it's not, it must refer to either an existing TDirectory inside the file or to a non-existing object (it may + // also be an existing object if --replace was passed). + TKey *destDirKey = nullptr; + if (!destPath.empty()) { + destDirKey = destFile->GetKey(std::string(destPath).c_str()); + if (destDirKey && !TClass::GetClass(destDirKey->GetClassName())->InheritsFrom("TDirectory")) { + if (!args.fReplace) { + Err() << "destination path \"" << destFname << ":" << destPath << "\" already exists (as an object of type " + << destDirKey->GetClassName() << "). Use the --replace flag to overwrite it.\n"; + return 1; + } else { + destDirKey = nullptr; + } + } + } + + // If we are copying multiple objects the destination path must either be empty or a TDirectory. + const bool destIsNewObject = !destPath.empty() && !destDirKey; + if (destIsNewObject && args.fSources.size() > 1) { + Err() << "multiple sources were specified, but destination path \"" << destFname << ":" << destPath + << "\" is not a directory.\n"; + return 1; + } + + if (args.fCompression) + destFile->SetCompressionSettings(*args.fCompression); + + const std::uint32_t flags = args.fRecursive * EGetMatchingPathsFlags::kRecursive; + // const bool oneFile = args.fSources.size() == 1; + for (const auto &srcFname : args.fSources) { + auto src = ROOT::CmdLine::ParseRootSource(srcFname, flags); + if (!src.fErrors.empty()) { + for (const auto &err : src.fErrors) + Err() << err << "\n"; + return 1; + } + + // If we are copying multiple objects the destination path must either be empty or a TDirectory. + if (destIsNewObject && src.fObjectTree.fLeafList.size() + src.fObjectTree.fDirList.size() > 1) { + Err() << "multiple sources were specified but destination path \"" << destFname << ":" << destPath + << "\" is not a directory.\n"; + return 1; + } + + // According to rootcp.py, this is necessary for fast copy. TODO: verify + gROOT->GetListOfFiles()->Remove(src.fObjectTree.fFile.get()); + + // Iterate all objects we need to copy + RootCpDestination dest; + dest.fFile = destFile.get(); + dest.fFname = destFname; + dest.fIsNewObject = destIsNewObject; + dest.fPath = destPath; + for (auto nodeIdx : src.fObjectTree.fLeafList) { + CopyNode(src, dest, nodeIdx, args); + } + for (auto nodeIdx : src.fObjectTree.fDirList) { + if (nodeIdx == 0) { + // The root file node needs special treatment; for all other "top-level" directories, CopyNode handles them. + const auto &node = src.fObjectTree.fNodes[nodeIdx]; + for (auto childIdx = node.fFirstChild; childIdx < node.fFirstChild + node.fNChildren; ++childIdx) + CopyNode(src, dest, childIdx, args); + } else { + CopyNode(src, dest, nodeIdx, args); + } + } + } + + return 0; +} diff --git a/main/src/rootls.cxx b/main/src/rootls.cxx index 4b0bcd0d47caa..8e3c65b8b2ba2 100644 --- a/main/src/rootls.cxx +++ b/main/src/rootls.cxx @@ -24,7 +24,8 @@ #include "logging.hxx" #include "optparse.hxx" -#include "wildcards.hpp" +#include "RootObjTree.hxx" +#include "RootObjTree.cxx" #include #include @@ -47,6 +48,8 @@ #undef GetClassName #endif +using namespace ROOT::CmdLine; + static const char *const kAnsiNone = "\x1B[0m"; static const char *const kAnsiGreen = "\x1B[32m"; static const char *const kAnsiBlue = "\x1B[34m"; @@ -115,44 +118,6 @@ static bool ClassInheritsFrom(const char *class_, const char *baseClass) return inherits; } -using NodeIdx = std::uint32_t; - -struct RootLsNode { - std::string fName; - std::string fClassName; - TKey *fKey = nullptr; // This is non-null for all nodes except the root node (which is the file itself) - - TDirectory *fDir = nullptr; // This is null for all non-directory nodes - // NOTE: by construction of the tree, all children of the same node are contiguous. - NodeIdx fFirstChild = 0; - std::uint32_t fNChildren = 0; - std::uint32_t fNesting = 0; - NodeIdx fParent = 0; -}; - -static RootLsNode NodeFromKey(TKey &key) -{ - RootLsNode node = {}; - node.fName = key.GetName(); - node.fClassName = key.GetClassName(); - node.fKey = &key; - return node; -} - -struct RootLsTree { - // 0th node is the root node - std::vector fNodes; - std::vector fDirList; - std::vector fLeafList; - // The file must be kept alive in order to access the nodes' keys - std::unique_ptr fFile; -}; - -struct RootLsSource { - std::string fFileName; - RootLsTree fObjectTree; -}; - struct RootLsArgs { enum EFlags { kNone = 0x0, @@ -170,7 +135,7 @@ struct RootLsArgs { }; std::uint32_t fFlags = 0; - std::vector fSources; + std::vector fSources; EPrintUsage fPrintUsageAndExit = EPrintUsage::kNo; }; @@ -303,7 +268,7 @@ static void PrintRNTuple(std::ostream &stream, const ROOT::RNTupleDescriptor &de } } -static void PrintChildrenDetailed(std::ostream &stream, const RootLsTree &tree, NodeIdx nodeIdx, std::uint32_t flags, +static void PrintChildrenDetailed(std::ostream &stream, const RootObjTree &tree, NodeIdx_t nodeIdx, std::uint32_t flags, Indent indent, std::size_t minNameLen = 0, std::size_t minClassLen = 0); /// Prints a `ls -l`-like output: @@ -320,9 +285,9 @@ static void PrintChildrenDetailed(std::ostream &stream, const RootLsTree &tree, /// \param nodesEnd The last node to be printed /// \param flags A bitmask of RootLsArgs::Flags that influence how stuff is printed /// \param indent Each line of the output will have these many leading whitespaces -static void PrintNodesDetailed(std::ostream &stream, const RootLsTree &tree, - std::vector::const_iterator nodesBegin, - std::vector::const_iterator nodesEnd, std::uint32_t flags, Indent indent, +static void PrintNodesDetailed(std::ostream &stream, const RootObjTree &tree, + std::vector::const_iterator nodesBegin, + std::vector::const_iterator nodesEnd, std::uint32_t flags, Indent indent, std::size_t minNameLen = 0, std::size_t minClassLen = 0) { std::size_t maxClassLen = 0, maxNameLen = 0; @@ -335,7 +300,7 @@ static void PrintNodesDetailed(std::ostream &stream, const RootLsTree &tree, maxNameLen = std::max(minNameLen, maxNameLen + 2); for (auto childIt = nodesBegin; childIt != nodesEnd; ++childIt) { - NodeIdx childIdx = *childIt; + NodeIdx_t childIdx = *childIt; const auto &child = tree.fNodes[childIdx]; const char *cycleStr = ""; @@ -392,7 +357,7 @@ static void PrintNodesDetailed(std::ostream &stream, const RootLsTree &tree, } /// \param nodeIdx The index of the node whose children should be printed -static void PrintChildrenDetailed(std::ostream &stream, const RootLsTree &tree, NodeIdx nodeIdx, std::uint32_t flags, +static void PrintChildrenDetailed(std::ostream &stream, const RootObjTree &tree, NodeIdx_t nodeIdx, std::uint32_t flags, Indent indent, std::size_t minNameLen, std::size_t minClassLen) { @@ -400,19 +365,19 @@ static void PrintChildrenDetailed(std::ostream &stream, const RootLsTree &tree, if (node.fNChildren == 0) return; - std::vector children(node.fNChildren); + std::vector children(node.fNChildren); std::iota(children.begin(), children.end(), node.fFirstChild); PrintNodesDetailed(stream, tree, children.begin(), children.end(), flags, indent, minNameLen, minClassLen); } // Prints all children of `nodeIdx`-th node in a ls-like fashion. -static void PrintChildrenInColumns(std::ostream &stream, const RootLsTree &tree, NodeIdx nodeIdx, std::uint32_t flags, - Indent indent); +static void PrintChildrenInColumns(std::ostream &stream, const RootObjTree &tree, NodeIdx_t nodeIdx, + std::uint32_t flags, Indent indent); // Prints a `ls`-like output -static void PrintNodesInColumns(std::ostream &stream, const RootLsTree &tree, - std::vector::const_iterator nodesBegin, - std::vector::const_iterator nodesEnd, std::uint32_t flags, Indent indent) +static void PrintNodesInColumns(std::ostream &stream, const RootObjTree &tree, + std::vector::const_iterator nodesBegin, + std::vector::const_iterator nodesEnd, std::uint32_t flags, Indent indent) { const auto nNodes = std::distance(nodesBegin, nodesEnd); if (nNodes == 0) @@ -422,7 +387,7 @@ static void PrintNodesInColumns(std::ostream &stream, const RootLsTree &tree, V2i terminalSize = GetTerminalSize(); terminalSize.x -= indent; const auto [minElemWidthIt, maxElemWidthIt] = - std::minmax_element(nodesBegin, nodesEnd, [&tree](NodeIdx aIdx, NodeIdx bIdx) { + std::minmax_element(nodesBegin, nodesEnd, [&tree](NodeIdx_t aIdx, NodeIdx_t bIdx) { const auto &a = tree.fNodes[aIdx]; const auto &b = tree.fNodes[bIdx]; return a.fName.length() < b.fName.length(); @@ -449,8 +414,8 @@ static void PrintNodesInColumns(std::ostream &stream, const RootLsTree &tree, int width = 0; for (auto j = 0u; j < nNodes; ++j) { if ((j % nCols) == colIdx) { - NodeIdx childIdx = nodesBegin[j]; - const RootLsNode &child = tree.fNodes[childIdx]; + NodeIdx_t childIdx = nodesBegin[j]; + const RootObjNode &child = tree.fNodes[childIdx]; width = std::max(width, child.fName.length() + minCharsBetween); } } @@ -479,7 +444,7 @@ static void PrintNodesInColumns(std::ostream &stream, const RootLsTree &tree, auto curCol = 0u; for (auto i = 0u; i < nNodes; ++i) { - NodeIdx childIdx = nodesBegin[i]; + NodeIdx_t childIdx = nodesBegin[i]; const auto &child = tree.fNodes[childIdx]; if (curCol == 0) { PrintIndent(stream, indent); @@ -503,7 +468,7 @@ static void PrintNodesInColumns(std::ostream &stream, const RootLsTree &tree, bool nextIsDirWithRecursiveDisplay = false; if ((flags & RootLsArgs::kRecursiveListing) && i < nNodes - 1) { - NodeIdx nextChildIdx = nodesBegin[i + 1]; + NodeIdx_t nextChildIdx = nodesBegin[i + 1]; const auto &nextChild = tree.fNodes[nextChildIdx]; nextIsDirWithRecursiveDisplay = nextChild.fNChildren > 0 && ClassInheritsFrom(nextChild.fClassName.c_str(), "TDirectory"); @@ -532,38 +497,18 @@ static void PrintNodesInColumns(std::ostream &stream, const RootLsTree &tree, } // Prints all children of `nodeIdx`-th node in a ls-like fashion. -static void PrintChildrenInColumns(std::ostream &stream, const RootLsTree &tree, NodeIdx nodeIdx, std::uint32_t flags, - Indent indent) +static void PrintChildrenInColumns(std::ostream &stream, const RootObjTree &tree, NodeIdx_t nodeIdx, + std::uint32_t flags, Indent indent) { const auto &node = tree.fNodes[nodeIdx]; if (node.fNChildren == 0) return; - std::vector children(node.fNChildren); + std::vector children(node.fNChildren); std::iota(children.begin(), children.end(), node.fFirstChild); PrintNodesInColumns(stream, tree, children.begin(), children.end(), flags, indent); } -static std::string NodeFullPath(const RootLsTree &tree, NodeIdx nodeIdx) -{ - std::vector fragments; - const RootLsNode *node = &tree.fNodes[nodeIdx]; - NodeIdx prevParent; - do { - prevParent = node->fParent; - fragments.push_back(&node->fName); - node = &tree.fNodes[node->fParent]; - } while (node->fParent != prevParent); - - assert(!fragments.empty()); - - std::string fullPath = **fragments.rbegin(); - for (auto it = std::next(fragments.rbegin()), end = fragments.rend(); it != end; ++it) { - fullPath += '/' + **it; - } - return fullPath; -} - // Main entrypoint of the program static void RootLs(const RootLsArgs &args, std::ostream &stream = std::cout) { @@ -582,7 +527,7 @@ static void RootLs(const RootLsArgs &args, std::ostream &stream = std::cout) const bool manySources = source.fObjectTree.fDirList.size() + source.fObjectTree.fLeafList.size() > 1; const Indent indent = outerIndent + manySources * 2; - for (NodeIdx rootIdx : source.fObjectTree.fDirList) { + for (NodeIdx_t rootIdx : source.fObjectTree.fDirList) { if (manySources) { PrintIndent(stream, outerIndent); stream << NodeFullPath(source.fObjectTree, rootIdx) << " :\n"; @@ -596,100 +541,6 @@ static void RootLs(const RootLsArgs &args, std::ostream &stream = std::cout) } } -static bool MatchesGlob(std::string_view haystack, std::string_view pattern) -{ - return wildcards::match(haystack, pattern); -} - -/// Inspects `fileName` to match all children that match `pattern`. Returns a tree with all the matched nodes. -/// `flags` is a bitmask of `RootLsArgs::Flags`. -static RootLsTree GetMatchingPathsInFile(std::string_view fileName, std::string_view pattern, std::uint32_t flags) -{ - RootLsTree nodeTree; - nodeTree.fFile = std::unique_ptr(TFile::Open(std::string(fileName).c_str(), "READ")); - if (!nodeTree.fFile) - return nodeTree; - - const auto patternSplits = pattern.empty() ? std::vector{} : ROOT::Split(pattern, "/"); - - // Match all objects at all nesting levels down to the deepest nesting level of `pattern` (or all nesting levels - // if we have the "recursive listing" flag). The nodes are visited breadth-first. - { - RootLsNode rootNode = {}; - rootNode.fName = std::string(fileName); - rootNode.fClassName = nodeTree.fFile->Class()->GetName(); - rootNode.fDir = nodeTree.fFile.get(); - nodeTree.fNodes.emplace_back(std::move(rootNode)); - } - std::deque nodesToVisit{0}; - - // Keep track of the object names found at every nesting level and only add the first one. - std::unordered_set namesFound; - - const bool isRecursive = flags & RootLsArgs::kRecursiveListing; - do { - NodeIdx curIdx = nodesToVisit.front(); - nodesToVisit.pop_front(); - RootLsNode *cur = &nodeTree.fNodes[curIdx]; - assert(cur->fDir); - - // Sort the keys by name - std::vector keys; - keys.reserve(cur->fDir->GetListOfKeys()->GetEntries()); - for (TKey *key : ROOT::Detail::TRangeStaticCast(cur->fDir->GetListOfKeys())) - keys.push_back(key); - - std::sort(keys.begin(), keys.end(), - [](const auto *a, const auto *b) { return strcmp(a->GetName(), b->GetName()) < 0; }); - - namesFound.clear(); - - for (TKey *key : keys) { - // Don't recurse lower than requested by `pattern` unless we explicitly have the `recursive listing` flag. - if (cur->fNesting < patternSplits.size() && !MatchesGlob(key->GetName(), patternSplits[cur->fNesting])) - continue; - - if (namesFound.count(key->GetName()) > 0) { - std::cerr << "WARNING: Several versions of '" << key->GetName() << "' are present in '" << fileName - << "'. Only the most recent will be considered.\n"; - continue; - } - namesFound.insert(key->GetName()); - - auto &newChild = nodeTree.fNodes.emplace_back(NodeFromKey(*key)); - // Need to get back cur since the emplace_back() may have moved it. - cur = &nodeTree.fNodes[curIdx]; - newChild.fNesting = cur->fNesting + 1; - newChild.fParent = curIdx; - if (!cur->fNChildren) - cur->fFirstChild = nodeTree.fNodes.size() - 1; - cur->fNChildren++; - - if (ClassInheritsFrom(key->GetClassName(), "TDirectory")) - newChild.fDir = cur->fDir->GetDirectory(key->GetName()); - } - - // Only recurse into subdirectories that are up to the deepest level we ask for through `pattern`. - if (cur->fNesting < patternSplits.size() || isRecursive) { - for (auto childIdx = cur->fFirstChild; childIdx < cur->fFirstChild + cur->fNChildren; ++childIdx) { - auto &child = nodeTree.fNodes[childIdx]; - if (child.fDir) - nodesToVisit.push_back(childIdx); - else if (cur->fNesting < patternSplits.size()) - nodeTree.fLeafList.push_back(childIdx); - } - } - if (cur->fNesting == patternSplits.size()) { - if (cur->fDir) - nodeTree.fDirList.push_back(curIdx); - else - nodeTree.fLeafList.push_back(curIdx); - } - } while (!nodesToVisit.empty()); - - return nodeTree; -} - static RootLsArgs ParseArgs(const char **args, int nArgs) { RootLsArgs outArgs; @@ -722,32 +573,8 @@ static RootLsArgs ParseArgs(const char **args, int nArgs) outArgs.fFlags |= opts.GetSwitch("rntupleListing") * RootLsArgs::kRNTupleListing; // Positional arguments - for (const auto &argStr : opts.GetArgs()) { - const char *arg = argStr.c_str(); - RootLsSource &newSource = outArgs.fSources.emplace_back(); - - // Handle known URI prefixes - static const char *const specialPrefixes[] = {"http", "https", "root", "gs", "s3"}; - for (const char *prefix : specialPrefixes) { - const auto prefixLen = strlen(prefix); - if (strncmp(arg, prefix, prefixLen) == 0 && strncmp(arg + prefixLen, "://", 3) == 0) { - newSource.fFileName = std::string(prefix) + "://"; - arg += prefixLen + 3; - break; - } - } - - auto tokens = ROOT::Split(arg, ":"); - if (tokens.empty()) - continue; - - newSource.fFileName += tokens[0]; - if (tokens.size() > 1) { - newSource.fObjectTree = GetMatchingPathsInFile(newSource.fFileName, tokens[1], outArgs.fFlags); - } else { - newSource.fObjectTree = GetMatchingPathsInFile(newSource.fFileName, "", outArgs.fFlags); - } - } + auto flags = !!(outArgs.fFlags & RootLsArgs::kRecursiveListing) * EGetMatchingPathsFlags::kRecursive; + outArgs.fSources = ROOT::CmdLine::ParseRootSources(opts.GetArgs(), flags); return outArgs; } @@ -776,7 +603,7 @@ int main(int argc, char **argv) // sort leaves by name for (auto &source : args.fSources) { std::sort(source.fObjectTree.fLeafList.begin(), source.fObjectTree.fLeafList.end(), - [&tree = source.fObjectTree](NodeIdx aIdx, NodeIdx bIdx) { + [&tree = source.fObjectTree](NodeIdx_t aIdx, NodeIdx_t bIdx) { const auto &a = tree.fNodes[aIdx]; const auto &b = tree.fNodes[bIdx]; return a.fName < b.fName; diff --git a/roottest/main/CMakeLists.txt b/roottest/main/CMakeLists.txt index 5a9bea3c8d9fc..ebedeb345f5db 100644 --- a/roottest/main/CMakeLists.txt +++ b/roottest/main/CMakeLists.txt @@ -136,7 +136,7 @@ ROOTTEST_ADD_TEST(SimplePattern3 ############################## ROORM TESTS ############################## ROOTTEST_ADD_TEST(SimpleRootrm1PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root victim1.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root victim1.root FIXTURES_SETUP main-SimpleRootrm1PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootrm1 @@ -158,7 +158,7 @@ ROOTTEST_ADD_TEST(SimpleRootrm1Clean ROOTTEST_ADD_TEST(SimpleRootrm2PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root victim2.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root victim2.root FIXTURES_SETUP main-SimpleRootrm2PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootrm2 @@ -180,7 +180,7 @@ ROOTTEST_ADD_TEST(SimpleRootrm2Clean ROOTTEST_ADD_TEST(SimpleRootrm3PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root victim3.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root victim3.root FIXTURES_SETUP main-SimpleRootrm3PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootrm3 @@ -202,7 +202,7 @@ ROOTTEST_ADD_TEST(SimpleRootrm3Clean ############################# ROOMKDIR TESTS ############################ ROOTTEST_ADD_TEST(SimpleRootmkdir1PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root target1.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root target1.root FIXTURES_SETUP main-SimpleRootmkdir1PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootmkdir1 @@ -224,7 +224,7 @@ ROOTTEST_ADD_TEST(SimpleRootmkdir1Clean ROOTTEST_ADD_TEST(SimpleRootmkdir2PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root target2.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root target2.root FIXTURES_SETUP main-SimpleRootmkdir2PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootmkdir2 @@ -246,7 +246,7 @@ ROOTTEST_ADD_TEST(SimpleRootmkdir2Clean ROOTTEST_ADD_TEST(SimpleRootmkdir3PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root target3.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root target3.root FIXTURES_SETUP main-SimpleRootmkdir3PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootmkdir3 @@ -266,13 +266,13 @@ ROOTTEST_ADD_TEST(SimpleRootmkdir3Clean FIXTURES_REQUIRED main-SimpleRootmkdir3CheckOutput-fixture) ######################################################################### -############################# ROOCP TESTS ############################ +############################# ROOTCP TESTS ############################ ROOTTEST_ADD_TEST(SimpleRootcp1PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root copy1.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root copy1.root FIXTURES_SETUP main-SimpleRootcp1PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootcp1 - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} copy1.root:hpx copy1.root:histo + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} copy1.root:hpx copy1.root:histo FIXTURES_REQUIRED main-SimpleRootcp1PrepareInput-fixture FIXTURES_SETUP main-SimpleRootcp1-fixture) @@ -290,11 +290,11 @@ ROOTTEST_ADD_TEST(SimpleRootcp1Clean ROOTTEST_ADD_TEST(SimpleRootcp2PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root copy2.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root copy2.root FIXTURES_SETUP main-SimpleRootcp2PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootcp2 - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} -r copy2.root:tof copy2.root:fot + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} -r copy2.root:tof copy2.root:fot FIXTURES_REQUIRED main-SimpleRootcp2PrepareInput-fixture FIXTURES_SETUP main-SimpleRootcp2-fixture) @@ -312,11 +312,11 @@ ROOTTEST_ADD_TEST(SimpleRootcp2Clean ROOTTEST_ADD_TEST(SimpleRootcp3PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root copy3.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root copy3.root FIXTURES_SETUP main-SimpleRootcp3PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootcp3 - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --replace copy3.root:hpx copy3.root:hpxpy + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --replace copy3.root:hpx copy3.root:hpxpy FIXTURES_REQUIRED main-SimpleRootcp3PrepareInput-fixture FIXTURES_SETUP main-SimpleRootcp3-fixture) @@ -333,11 +333,11 @@ ROOTTEST_ADD_TEST(SimpleRootcp3Clean ROOTTEST_ADD_TEST(SimpleRootcp4PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root copy4.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root copy4.root FIXTURES_SETUP main-SimpleRootcp4PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootcp4 - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} copy4.root:hpx copy4.root:dir + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} copy4.root:hpx copy4.root:dir FIXTURES_REQUIRED main-SimpleRootcp4PrepareInput-fixture FIXTURES_SETUP main-SimpleRootcp4-fixture) @@ -355,11 +355,11 @@ ROOTTEST_ADD_TEST(SimpleRootcp4Clean ROOTTEST_ADD_TEST(SimpleRootcp5PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root copy5.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root copy5.root FIXTURES_SETUP main-SimpleRootcp5PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootcp5 - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} -r copy5.root:tof copy5.root:dir + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} -r copy5.root:tof copy5.root:dir FIXTURES_REQUIRED main-SimpleRootcp5PrepareInput-fixture FIXTURES_SETUP main-SimpleRootcp5-fixture) @@ -377,7 +377,7 @@ ROOTTEST_ADD_TEST(SimpleRootcp5Clean ############################# ROOMV TESTS ############################ ROOTTEST_ADD_TEST(SimpleRootmv1PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root move1.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root move1.root FIXTURES_SETUP main-SimpleRootmv1PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootmv1 @@ -399,7 +399,7 @@ ROOTTEST_ADD_TEST(SimpleRootmv1Clean ROOTTEST_ADD_TEST(SimpleRootmv2PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root move2.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root move2.root FIXTURES_SETUP main-SimpleRootmv2PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootmv2 @@ -421,7 +421,7 @@ ROOTTEST_ADD_TEST(SimpleRootmv2Clean ROOTTEST_ADD_TEST(SimpleRootmv3PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root move3.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root move3.root FIXTURES_SETUP main-SimpleRootmv3PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootmv3 @@ -442,7 +442,7 @@ ROOTTEST_ADD_TEST(SimpleRootmv3Clean ROOTTEST_ADD_TEST(SimpleRootmv4PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root move4.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root move4.root FIXTURES_SETUP main-SimpleRootmv4PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootmv4 @@ -463,7 +463,7 @@ ROOTTEST_ADD_TEST(SimpleRootmv4Clean ROOTTEST_ADD_TEST(SimpleRootmv5PrepareInput - COMMAND ${PY_TOOLS_PREFIX}/rootcp${pyext} --recreate -r test.root move5.root + COMMAND ${TOOLS_PREFIX}/rootcp${exeext} --recreate -r test.root move5.root FIXTURES_SETUP main-SimpleRootmv5PrepareInput-fixture) ROOTTEST_ADD_TEST(SimpleRootmv5