Skip to content

Commit ea7d908

Browse files
gave92gave92
authored
Improve folder enumeration speed (#4922)
1 parent de5dd1b commit ea7d908

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+122
-1103
lines changed

‎Files/Files.csproj

-1
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,6 @@
261261
<Compile Include="Helpers\CollectionDebugView.cs" />
262262
<Compile Include="Helpers\DynamicDialogFactory.cs" />
263263
<Compile Include="Helpers\Extension.cs" />
264-
<Compile Include="Helpers\FileListCache\CacheEntry.cs" />
265264
<Compile Include="Helpers\FileListCache\FileListCacheController.cs" />
266265
<Compile Include="Helpers\FileListCache\IFileListCache.cs" />
267266
<Compile Include="Helpers\FileListCache\PersistentSQLiteCacheAdapter.cs" />

‎Files/Filesystem/StorageEnumerators/UniversalStorageEnumerator.cs

+2-17
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ public static async Task<List<ListedItem>> ListEntries(
2323
string returnformat,
2424
Type sourcePageType,
2525
CancellationToken cancellationToken,
26-
List<string> skipItems,
2726
int countLimit,
2827
Func<List<ListedItem>, Task> intermediateAction
2928
)
@@ -74,14 +73,7 @@ ex is UnauthorizedAccessException
7473
var folder = await AddFolderAsync(item as StorageFolder, currentStorageFolder, returnformat, cancellationToken);
7574
if (folder != null)
7675
{
77-
if (skipItems?.Contains(folder.ItemPath) ?? false)
78-
{
79-
skipItems.Remove(folder.ItemPath);
80-
}
81-
else
82-
{
83-
tempList.Add(folder);
84-
}
76+
tempList.Add(folder);
8577
}
8678
}
8779
else
@@ -90,14 +82,7 @@ ex is UnauthorizedAccessException
9082
var fileEntry = await AddFileAsync(file, currentStorageFolder, returnformat, true, sourcePageType, cancellationToken);
9183
if (fileEntry != null)
9284
{
93-
if (skipItems?.Contains(fileEntry.ItemPath) ?? false)
94-
{
95-
skipItems.Remove(fileEntry.ItemPath);
96-
}
97-
else
98-
{
99-
tempList.Add(fileEntry);
100-
}
85+
tempList.Add(fileEntry);
10186
}
10287
}
10388
if (cancellationToken.IsCancellationRequested)

‎Files/Filesystem/StorageEnumerators/Win32StorageEnumerator.cs

+2-17
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ public static async Task<List<ListedItem>> ListEntries(
2828
WIN32_FIND_DATA findData,
2929
NamedPipeAsAppServiceConnection connection,
3030
CancellationToken cancellationToken,
31-
List<string> skipItems,
3231
int countLimit,
3332
Func<List<ListedItem>, Task> intermediateAction
3433
)
@@ -49,14 +48,7 @@ Func<List<ListedItem>, Task> intermediateAction
4948
var file = await GetFile(findData, path, returnformat, connection, cancellationToken);
5049
if (file != null)
5150
{
52-
if (skipItems?.Contains(file.ItemPath) ?? false)
53-
{
54-
skipItems.Remove(file.ItemPath);
55-
}
56-
else
57-
{
58-
tempList.Add(file);
59-
}
51+
tempList.Add(file);
6052
++count;
6153
}
6254
}
@@ -67,14 +59,7 @@ Func<List<ListedItem>, Task> intermediateAction
6759
var folder = await GetFolder(findData, path, returnformat, cancellationToken);
6860
if (folder != null)
6961
{
70-
if (skipItems?.Contains(folder.ItemPath) ?? false)
71-
{
72-
skipItems.Remove(folder.ItemPath);
73-
}
74-
else
75-
{
76-
tempList.Add(folder);
77-
}
62+
tempList.Add(folder);
7863
++count;
7964
}
8065
}

‎Files/Helpers/FileListCache/CacheEntry.cs

-11
This file was deleted.

‎Files/Helpers/FileListCache/FileListCacheController.cs

-48
Original file line numberDiff line numberDiff line change
@@ -20,59 +20,11 @@ private FileListCacheController()
2020
persistentAdapter = new PersistentSQLiteCacheAdapter();
2121
}
2222

23-
private readonly IMemoryCache filesCache = new MemoryCache(new MemoryCacheOptions
24-
{
25-
SizeLimit = 1_000_000
26-
});
27-
2823
private readonly IMemoryCache fileNamesCache = new MemoryCache(new MemoryCacheOptions
2924
{
3025
SizeLimit = 1_000_000
3126
});
3227

33-
public Task SaveFileListToCache(string path, CacheEntry cacheEntry)
34-
{
35-
if (!App.AppSettings.UseFileListCache)
36-
{
37-
return Task.CompletedTask;
38-
}
39-
40-
if (cacheEntry == null)
41-
{
42-
filesCache.Remove(path);
43-
return persistentAdapter.SaveFileListToCache(path, cacheEntry);
44-
}
45-
filesCache.Set(path, cacheEntry, new MemoryCacheEntryOptions
46-
{
47-
Size = cacheEntry.FileList.Count
48-
});
49-
50-
// save entry to persistent cache in background
51-
return persistentAdapter.SaveFileListToCache(path, cacheEntry);
52-
}
53-
54-
public async Task<CacheEntry> ReadFileListFromCache(string path, CancellationToken cancellationToken)
55-
{
56-
if (!App.AppSettings.UseFileListCache)
57-
{
58-
return null;
59-
}
60-
61-
var entry = filesCache.Get<CacheEntry>(path);
62-
if (entry == null)
63-
{
64-
entry = await persistentAdapter.ReadFileListFromCache(path, cancellationToken);
65-
if (entry?.FileList != null)
66-
{
67-
filesCache.Set(path, entry, new MemoryCacheEntryOptions
68-
{
69-
Size = entry.FileList.Count
70-
});
71-
}
72-
}
73-
return entry;
74-
}
75-
7628
public async Task<string> ReadFileDisplayNameFromCache(string path, CancellationToken cancellationToken)
7729
{
7830
var displayName = fileNamesCache.Get<string>(path);

‎Files/Helpers/FileListCache/IFileListCache.cs

-4
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,6 @@ namespace Files.Helpers.FileListCache
55
{
66
internal interface IFileListCache
77
{
8-
public Task<CacheEntry> ReadFileListFromCache(string path, CancellationToken cancellationToken);
9-
10-
public Task SaveFileListToCache(string path, CacheEntry cacheEntry);
11-
128
public Task<string> ReadFileDisplayNameFromCache(string path, CancellationToken cancellationToken);
139

1410
public Task SaveFileDisplayNameToCache(string path, string displayName);

‎Files/Helpers/FileListCache/PersistentSQLiteCacheAdapter.cs

-123
Original file line numberDiff line numberDiff line change
@@ -15,98 +15,6 @@ internal class PersistentSQLiteCacheAdapter : IFileListCache, IDisposable
1515
private SqliteConnection connection;
1616
private bool disposedValue;
1717

18-
public async Task SaveFileListToCache(string path, CacheEntry cacheEntry)
19-
{
20-
if (!await InitializeIfNeeded())
21-
{
22-
return;
23-
}
24-
const int maxCachedEntries = 128;
25-
try
26-
{
27-
if (cacheEntry == null)
28-
{
29-
using var deleteCommand = new SqliteCommand("DELETE FROM FileListCache WHERE Id = @Id", connection);
30-
deleteCommand.Parameters.Add("@Id", SqliteType.Text).Value = path;
31-
await deleteCommand.ExecuteNonQueryAsync();
32-
return;
33-
}
34-
35-
if (cacheEntry.FileList.Count > maxCachedEntries)
36-
{
37-
cacheEntry.FileList = cacheEntry.FileList.Take(maxCachedEntries).ToList();
38-
}
39-
40-
using var cmd = new SqliteCommand("SELECT Id FROM FileListCache WHERE Id = @Id", connection);
41-
cmd.Parameters.Add("@Id", SqliteType.Text).Value = path;
42-
using var reader = await cmd.ExecuteReaderAsync();
43-
if (reader.HasRows)
44-
{
45-
// need to update entry
46-
using var updateCommand = new SqliteCommand("UPDATE FileListCache SET Timestamp = @Timestamp, Entry = @Entry WHERE Id = @Id", connection);
47-
updateCommand.Parameters.Add("@Id", SqliteType.Text).Value = path;
48-
updateCommand.Parameters.Add("@Timestamp", SqliteType.Integer).Value = GetTimestamp(DateTime.UtcNow);
49-
var settings = new JsonSerializerSettings
50-
{
51-
TypeNameHandling = TypeNameHandling.Auto
52-
};
53-
updateCommand.Parameters.Add("@Entry", SqliteType.Text).Value = JsonConvert.SerializeObject(cacheEntry, settings);
54-
await updateCommand.ExecuteNonQueryAsync();
55-
}
56-
else
57-
{
58-
// need to insert entry
59-
using var insertCommand = new SqliteCommand("INSERT INTO FileListCache (Id, Timestamp, Entry) VALUES (@Id, @Timestamp, @Entry)", connection);
60-
insertCommand.Parameters.Add("@Id", SqliteType.Text).Value = path;
61-
insertCommand.Parameters.Add("@Timestamp", SqliteType.Integer).Value = GetTimestamp(DateTime.UtcNow);
62-
var settings = new JsonSerializerSettings
63-
{
64-
TypeNameHandling = TypeNameHandling.Auto
65-
};
66-
insertCommand.Parameters.Add("@Entry", SqliteType.Text).Value = JsonConvert.SerializeObject(cacheEntry, settings);
67-
await insertCommand.ExecuteNonQueryAsync();
68-
}
69-
}
70-
catch (Exception ex)
71-
{
72-
NLog.LogManager.GetCurrentClassLogger().Warn(ex, ex.Message);
73-
}
74-
}
75-
76-
public async Task<CacheEntry> ReadFileListFromCache(string path, CancellationToken cancellationToken)
77-
{
78-
if (!await InitializeIfNeeded())
79-
{
80-
return null;
81-
}
82-
try
83-
{
84-
using var cmd = new SqliteCommand("SELECT Timestamp, Entry FROM FileListCache WHERE Id = @Id", connection);
85-
cmd.Parameters.Add("@Id", SqliteType.Text).Value = path;
86-
87-
using var reader = await cmd.ExecuteReaderAsync(cancellationToken);
88-
if (!await reader.ReadAsync())
89-
{
90-
return null;
91-
}
92-
var timestamp = reader.GetInt64(0);
93-
var entryAsJson = reader.GetString(1);
94-
var settings = new JsonSerializerSettings
95-
{
96-
TypeNameHandling = TypeNameHandling.Auto
97-
};
98-
var entry = JsonConvert.DeserializeObject<CacheEntry>(entryAsJson, settings);
99-
entry.CurrentFolder.ItemPropertiesInitialized = false;
100-
entry.FileList.ForEach((item) => item.ItemPropertiesInitialized = false);
101-
return entry;
102-
}
103-
catch (Exception ex)
104-
{
105-
NLog.LogManager.GetCurrentClassLogger().Warn(ex, ex.Message);
106-
return null;
107-
}
108-
}
109-
11018
public async Task SaveFileDisplayNameToCache(string path, string displayName)
11119
{
11220
if (!await InitializeIfNeeded())
@@ -174,11 +82,6 @@ public async Task<string> ReadFileDisplayNameFromCache(string path, Cancellation
17482
}
17583
}
17684

177-
private long GetTimestamp(DateTime dateTime)
178-
{
179-
return new DateTimeOffset(dateTime).ToUnixTimeSeconds();
180-
}
181-
18285
public void Dispose()
18386
{
18487
if (!disposedValue)
@@ -190,23 +93,6 @@ public void Dispose()
19093

19194
private void RunCleanupRoutine()
19295
{
193-
Task.Run(async () =>
194-
{
195-
try
196-
{
197-
// remove entries that are 1 month old (timestamp is updated every time the cache is set)
198-
var limitTimestamp = GetTimestamp(DateTime.Now.AddMonths(-1));
199-
using var cmd = new SqliteCommand("DELETE FROM FileListCache WHERE Timestamp < @Timestamp", connection);
200-
cmd.Parameters.Add("@Timestamp", SqliteType.Integer).Value = limitTimestamp;
201-
202-
var count = await cmd.ExecuteNonQueryAsync().ConfigureAwait(false);
203-
Debug.WriteLine($"Removed {count} old entries from cache database");
204-
}
205-
catch (Exception ex)
206-
{
207-
NLog.LogManager.GetCurrentClassLogger().Warn(ex, ex.Message);
208-
}
209-
});
21096
}
21197

21298
private async Task<bool> InitializeIfNeeded()
@@ -227,15 +113,6 @@ private async Task<bool> InitializeIfNeeded()
227113
connection.Open();
228114

229115
// create db schema
230-
var createFileListCacheTable = @"CREATE TABLE IF NOT EXISTS ""FileListCache"" (
231-
""Id"" VARCHAR(5000) NOT NULL,
232-
""Timestamp"" INTEGER NOT NULL,
233-
""Entry"" TEXT NOT NULL,
234-
PRIMARY KEY(""Id"")
235-
)";
236-
using var cmdFileListCacheTable = new SqliteCommand(createFileListCacheTable, connection);
237-
cmdFileListCacheTable.ExecuteNonQuery();
238-
239116
var createFileDisplayNameCacheTable = @"CREATE TABLE IF NOT EXISTS ""FileDisplayNameCache"" (
240117
""Id"" VARCHAR(5000) NOT NULL,
241118
""DisplayName"" TEXT NOT NULL,

‎Files/MultilingualResources/Files.ar.xlf

-12
Original file line numberDiff line numberDiff line change
@@ -1845,10 +1845,6 @@
18451845
<source>Original path</source>
18461846
<target state="translated">المسار الأصلي</target>
18471847
</trans-unit>
1848-
<trans-unit id="SettingsUseFileListCache.Header" translate="yes" xml:space="preserve">
1849-
<source>Cache files and folders for better performance</source>
1850-
<target state="translated">ملفات ومجلدات ذاكرة التخزين المؤقت لأداء أفضل</target>
1851-
</trans-unit>
18521848
<trans-unit id="AppBarButtonAdd.Label" translate="yes" xml:space="preserve">
18531849
<source>Add</source>
18541850
<target state="translated" state-qualifier="tm-suggestion">إضافة</target>
@@ -2249,14 +2245,6 @@
22492245
<source>Disconnect</source>
22502246
<target state="needs-review-translation" state-qualifier="tm-suggestion">قطع الاتصال</target>
22512247
</trans-unit>
2252-
<trans-unit id="PreemptiveCacheParallelLimit.Header" translate="yes" xml:space="preserve">
2253-
<source>Preemptive cache parallel limit (smaller numbers should work better for hard drives)</source>
2254-
<target state="new">Preemptive cache parallel limit (smaller numbers should work better for hard drives)</target>
2255-
</trans-unit>
2256-
<trans-unit id="SettingsUsePreemptiveCache.Header" translate="yes" xml:space="preserve">
2257-
<source>Use preemptive cache (preload entries in child directories on navigation)</source>
2258-
<target state="new">Use preemptive cache (preload entries in child directories on navigation)</target>
2259-
</trans-unit>
22602248
<trans-unit id="BundlesWidgetMoreOptionsButton.AutomationProperties.Name" translate="yes" xml:space="preserve">
22612249
<source>More bundles options</source>
22622250
<target state="new">More bundles options</target>

‎Files/MultilingualResources/Files.cs-CZ.xlf

-12
Original file line numberDiff line numberDiff line change
@@ -1860,10 +1860,6 @@
18601860
<source>Original path</source>
18611861
<target state="translated">Umístění</target>
18621862
</trans-unit>
1863-
<trans-unit id="SettingsUseFileListCache.Header" translate="yes" xml:space="preserve">
1864-
<source>Cache files and folders for better performance</source>
1865-
<target state="translated">Indexovat soubory a složky pro lepší výkon</target>
1866-
</trans-unit>
18671863
<trans-unit id="AppBarButtonAdd.Label" translate="yes" xml:space="preserve">
18681864
<source>Add</source>
18691865
<target state="translated">Přidat</target>
@@ -2262,14 +2258,6 @@
22622258
<source>Disconnect</source>
22632259
<target state="needs-review-translation" state-qualifier="tm-suggestion">Odpojit</target>
22642260
</trans-unit>
2265-
<trans-unit id="PreemptiveCacheParallelLimit.Header" translate="yes" xml:space="preserve">
2266-
<source>Preemptive cache parallel limit (smaller numbers should work better for hard drives)</source>
2267-
<target state="new">Preemptive cache parallel limit (smaller numbers should work better for hard drives)</target>
2268-
</trans-unit>
2269-
<trans-unit id="SettingsUsePreemptiveCache.Header" translate="yes" xml:space="preserve">
2270-
<source>Use preemptive cache (preload entries in child directories on navigation)</source>
2271-
<target state="new">Use preemptive cache (preload entries in child directories on navigation)</target>
2272-
</trans-unit>
22732261
<trans-unit id="BundlesWidgetMoreOptionsButton.AutomationProperties.Name" translate="yes" xml:space="preserve">
22742262
<source>More bundles options</source>
22752263
<target state="new">More bundles options</target>

0 commit comments

Comments
 (0)