Skip cloud files marked as "not on disk" during command discovery (PowerShell#18152)

Author: SeeminglyScience

experimental-feature-linux.json

@@ -4,5 +4,6 @@
   "PSLoadAssemblyFromNativeCode",
   "PSNativeCommandErrorActionPreference",
   "PSSubsystemPluginModel",
+  "PSModuleAutoLoadSkipOfflineFiles",
   "PSFeedbackProvider"
 ]

experimental-feature-windows.json

@@ -4,5 +4,6 @@
   "PSLoadAssemblyFromNativeCode",
   "PSNativeCommandErrorActionPreference",
   "PSSubsystemPluginModel",
+  "PSModuleAutoLoadSkipOfflineFiles",
   "PSFeedbackProvider"
 ]

src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs

@@ -22,6 +22,7 @@ public class ExperimentalFeature
 
         internal const string EngineSource = "PSEngine";
         internal const string PSNativeCommandErrorActionPreferenceFeatureName = "PSNativeCommandErrorActionPreference";
+        internal const string PSModuleAutoLoadSkipOfflineFilesFeatureName = "PSModuleAutoLoadSkipOfflineFiles";
         internal const string PSCustomTableHeaderLabelDecoration = "PSCustomTableHeaderLabelDecoration";
         internal const string PSFeedbackProvider = "PSFeedbackProvider";
 
@@ -118,6 +119,9 @@ static ExperimentalFeature()
                 new ExperimentalFeature(
                     name: PSNativeCommandErrorActionPreferenceFeatureName,
                     description: "Native commands with non-zero exit codes issue errors according to $ErrorActionPreference when $PSNativeCommandUseErrorActionPreference is $true"),
+                new ExperimentalFeature(
+                    name: PSModuleAutoLoadSkipOfflineFilesFeatureName,
+                    description: "Module discovery will skip over files that are marked by cloud providers as not fully on disk."),
                 new ExperimentalFeature(
                     name: PSCustomTableHeaderLabelDecoration,
                     description: "Formatting differentiation for table header labels that aren't property members"),

src/System.Management.Automation/engine/Modules/ModuleUtils.cs

@@ -13,22 +13,46 @@ namespace System.Management.Automation.Internal
 {
     internal static class ModuleUtils
     {
+        // These are documented members FILE_ATTRIBUTE, they just have not yet been
+        // added to System.IO.FileAttributes yet.
+        private const int FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 0x400000;
+
+        private const int FILE_ATTRIBUTE_RECALL_ON_OPEN = 0x40000;
+
         // Default option for local file system enumeration:
         //  - Ignore files/directories when access is denied;
         //  - Search top directory only.
         private static readonly System.IO.EnumerationOptions s_defaultEnumerationOptions =
-                                        new System.IO.EnumerationOptions() { AttributesToSkip = FileAttributes.Hidden };
+                                        new System.IO.EnumerationOptions() { AttributesToSkip = FileAttributesToSkip };
+
+        private static readonly FileAttributes FileAttributesToSkip;
 
         // Default option for UNC path enumeration. Same as above plus a large buffer size.
         // For network shares, a large buffer may result in better performance as more results can be batched over the wire.
         // The buffer size 16K is recommended in the comment of the 'BufferSize' property:
         //    "A "large" buffer, for example, would be 16K. Typical is 4K."
         private static readonly System.IO.EnumerationOptions s_uncPathEnumerationOptions =
-                                        new System.IO.EnumerationOptions() { AttributesToSkip = FileAttributes.Hidden, BufferSize = 16384 };
+                                        new System.IO.EnumerationOptions() { AttributesToSkip = FileAttributesToSkip, BufferSize = 16384 };
 
         private static readonly string EnCulturePath = Path.DirectorySeparatorChar + "en";
         private static readonly string EnUsCulturePath = Path.DirectorySeparatorChar + "en-us";
 
+        static ModuleUtils()
+        {
+            if (ExperimentalFeature.IsEnabled(ExperimentalFeature.PSModuleAutoLoadSkipOfflineFilesFeatureName))
+            {
+                FileAttributesToSkip = FileAttributes.Hidden
+                    // Skip OneDrive files/directories that are not fully on disk.
+                    | FileAttributes.Offline
+                    | (FileAttributes)FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS
+                    | (FileAttributes)FILE_ATTRIBUTE_RECALL_ON_OPEN;
+
+                return;
+            }
+
+            FileAttributesToSkip = FileAttributes.Hidden;
+        }
+
         /// <summary>
         /// Check if a directory is likely a localized resources folder.
         /// </summary>
@@ -276,6 +300,11 @@ internal static IEnumerable<string> GetDefaultAvailableModuleFiles(string topDir
                     manifestPath += StringLiterals.PowerShellDataFileExtension;
                     if (File.Exists(manifestPath))
                     {
+                        if (HasSkippedFileAttribute(manifestPath))
+                        {
+                            continue;
+                        }
+
                         isModuleDirectory = true;
                         yield return manifestPath;
                     }
@@ -288,6 +317,11 @@ internal static IEnumerable<string> GetDefaultAvailableModuleFiles(string topDir
                         string moduleFile = Path.Combine(directoryToCheck, proposedModuleName) + ext;
                         if (File.Exists(moduleFile))
                         {
+                            if (HasSkippedFileAttribute(moduleFile))
+                            {
+                                continue;
+                            }
+
                             isModuleDirectory = true;
                             yield return moduleFile;
 
@@ -337,6 +371,25 @@ internal static List<Version> GetModuleVersionSubfolders(string moduleBase)
             return versionFolders;
         }
 
+        private static bool HasSkippedFileAttribute(string path)
+        {
+            try
+            {
+                FileAttributes attributes = File.GetAttributes(path);
+                if ((attributes & FileAttributesToSkip) is not 0)
+                {
+                    return true;
+                }
+            }
+            catch
+            {
+                // Ignore failures so that we keep the current behavior of failing
+                // later in the search.
+            }
+
+            return false;
+        }
+
         private static void ProcessPossibleVersionSubdirectories(IEnumerable<string> subdirectories, List<Version> versionFolders)
         {
             foreach (string subdir in subdirectories)