Implement new/create/upload/validate/list/delete commands

.gitignore

@@ -0,0 +1,9 @@
+/.idea
+_ReSharper.Caches/
+.vs/
+bin/
+obj/
+*.user
+/Subplatform.Snk
+/*.nupkg
+/*.nuspec

Core/Core.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net5.0</TargetFramework>
+    <AssemblyName>JetBrains.SymbolStorage</AssemblyName>
+    <RootNamespace>JetBrains.SymbolStorage</RootNamespace>
+    <SignAssembly>true</SignAssembly>
+    <AssemblyOriginatorKeyFile>..\Subplatform.Snk</AssemblyOriginatorKeyFile>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="AWSSDK.S3" Version="3.5.8.3" />
+    <PackageReference Include="JetBrains.Annotations" Version="2020.3.0" />
+    <PackageReference Include="Microsoft.Extensions.CommandLineUtils" Version="1.1.1" />
+    <PackageReference Include="Microsoft.SymbolStore" Version="1.0.210901" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="System.Linq.Async" Version="5.0.0" />
+    <PackageReference Include="MSFTCompressionCab" Version="1.0.0" />
+  </ItemGroup>
+</Project>

Core/src/AssemblyInfo.cs

@@ -0,0 +1,3 @@
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("JetBrains.SymbolStorage.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010087f63ba6a789c30e210e7ec987234ad9fe33baf7367993bab1b312d6f72ca296b91ed5c658964ffb9e7570eb184a527c68c6bdba41cfe67d8cfd3f888234206bf39205a3652d3af3445bb6f715fdac532e289fea41229bac37762b67eb16f58fee717d2465fca9ee17f08ed16772a1fc52c1c17022e1f0d9bdd004524a663aca")]

Core/src/Impl/Commands/ConsoleUtil.cs

@@ -0,0 +1,29 @@
+using System;
+using System.Text;
+using JetBrains.Annotations;
+
+namespace JetBrains.SymbolStorage.Impl.Commands
+{
+  internal static class ConsoleUtil
+  {
+    [NotNull]
+    public static string ReadHiddenConsoleInput([NotNull] string str)
+    {
+      Console.Write(str);
+      Console.Write(": ");
+      var secret = new StringBuilder();
+      while (true)
+      {
+        var key = Console.ReadKey(true);
+        if (key.Key == ConsoleKey.Enter)
+          break;
+        if (key.Key == ConsoleKey.Backspace && secret.Length > 0)
+          secret.Remove(secret.Length - 1, 1);
+        else if (key.Key != ConsoleKey.Backspace)
+          secret.Append(key.KeyChar);
+      }
+      Console.WriteLine();
+      return secret.ToString();
+    }
+  }
+}

Core/src/Impl/Commands/CreateCommand.cs

@@ -0,0 +1,199 @@
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Threading.Tasks;
+using JetBrains.Annotations;
+using JetBrains.SymbolStorage.Impl.Logger;
+using JetBrains.SymbolStorage.Impl.Storages;
+using JetBrains.SymbolStorage.Impl.Tags;
+using Microsoft.Deployment.Compression.Cab;
+
+namespace JetBrains.SymbolStorage.Impl.Commands
+{
+  internal sealed class CreateCommand
+  {
+    private readonly StorageFormat myExpectedStorageFormat;
+    private readonly bool myIsCompressPe;
+    private readonly bool myIsCompressWPdb;
+    private readonly bool myIsKeepNonCompressed;
+    private readonly ILogger myLogger;
+    private readonly string myProduct;
+    private readonly IEnumerable<string> myProperties;
+    private readonly IReadOnlyCollection<string> mySources;
+    private readonly IStorage myStorage;
+    private readonly string myToolId;
+    private readonly string myVersion;
+
+    public CreateCommand(
+      [NotNull] ILogger logger,
+      [NotNull] IStorage storage,
+      StorageFormat expectedStorageFormat,
+      [NotNull] string toolId,
+      [NotNull] string product,
+      [NotNull] string version,
+      bool isCompressPe,
+      bool isCompressWPdb,
+      bool isKeepNonCompressed,
+      [NotNull] IEnumerable<string> properties,
+      [NotNull] IReadOnlyCollection<string> sources)
+    {
+      myLogger = logger ?? throw new ArgumentNullException(nameof(logger));
+      myStorage = storage ?? throw new ArgumentNullException(nameof(storage));
+      myExpectedStorageFormat = expectedStorageFormat;
+      myToolId = toolId ?? throw new ArgumentNullException(nameof(toolId));
+      myProduct = product ?? throw new ArgumentNullException(nameof(product));
+      myVersion = version ?? throw new ArgumentNullException(nameof(version));
+      myIsCompressPe = isCompressPe;
+      myIsCompressWPdb = isCompressWPdb;
+      myIsKeepNonCompressed = isKeepNonCompressed;
+      myProperties = properties ?? throw new ArgumentNullException(nameof(properties));
+      mySources = sources ?? throw new ArgumentNullException(nameof(sources));
+    }
+
+    public async Task<int> Execute()
+    {
+      if (!myProduct.ValidateProduct())
+        throw new ApplicationException($"Invalid product name {myProduct}");
+      if (!myVersion.ValidateVersion())
+        throw new ApplicationException($"Invalid version {myVersion}");
+
+      await new Validator(myLogger, myStorage).CreateStorageMarkers(myExpectedStorageFormat);
+
+      var dirs = new ConcurrentBag<string>();
+      var scanner = new Scanner(myLogger, myIsCompressPe, myIsCompressWPdb, myIsKeepNonCompressed, mySources,
+        async (sourceDir, sourceRelativeFile, storageRelativeFile) =>
+          {
+            await WriteData(Path.Combine(sourceDir, sourceRelativeFile), storageRelativeFile, (file, len, stream) => myStorage.CreateForWriting(file, AccessMode.Public, len, stream));
+            dirs.Add(Path.GetDirectoryName(storageRelativeFile));
+          },
+        async (sourceDir, sourceRelativeFile, packedStorageRelativeFile) =>
+          {
+            await WriteDataPacked(Path.Combine(sourceDir, sourceRelativeFile), packedStorageRelativeFile, (file, len, stream) => myStorage.CreateForWriting(file, AccessMode.Public, len, stream));
+            dirs.Add(Path.GetDirectoryName(packedStorageRelativeFile));
+          });
+
+      var statistics = await scanner.Execute();
+      myLogger.Info($"[{DateTime.Now:s}] Done with data (warnings: {statistics.Warnings}, errors: {statistics.Errors})");
+      if (statistics.HasProblems)
+      {
+        myLogger.Error("Found some issues, creating was interrupted");
+        return 1;
+      }
+
+      await WriteTag(dirs);
+      return 0;
+    }
+
+    private async Task WriteTag([NotNull] IEnumerable<string> dirs)
+    {
+      myLogger.Info($"[{DateTime.Now:s}] Writing tag file...");
+      var fileId = Guid.NewGuid();
+      await using var stream = new MemoryStream();
+      TagUtil.WriteTagScript(new Tag
+        {
+          ToolId = myToolId,
+          FileId = fileId.ToString(),
+          Product = myProduct,
+          Version = myVersion,
+          Properties = myProperties.ToTagProperties(),
+          Directories = dirs.Distinct().OrderBy(x => x, StringComparer.Ordinal).ToArray()
+        }, stream);
+
+      var tagFile = Path.Combine(TagUtil.TagDirectory, myProduct, myProduct + '-' + myVersion + '-' + fileId.ToString("N") + TagUtil.TagExtension);
+      await myStorage.CreateForWriting(tagFile, AccessMode.Private, stream.Length, stream.Rewind());
+    }
+
+    private static async Task WriteData(
+      [NotNull] string sourceFile,
+      [NotNull] string storageRelativeFile,
+      [NotNull] Func<string, long, Stream, Task> writeStorageFile)
+    {
+      await using var stream = File.OpenRead(sourceFile);
+      await writeStorageFile(storageRelativeFile, stream.Length, stream);
+    }
+
+    private static async Task WriteDataPacked(
+      [NotNull] string sourceFile,
+      [NotNull] string packedStorageRelativeFile,
+      [NotNull] Func<string, long, Stream, Task> writeStorageFile)
+    {
+      if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+        throw new PlatformNotSupportedException("The Windows PDB and PE compression works only on Windows");
+      var tempFile = Path.GetTempFileName();
+      try
+      {
+        new CabInfo(tempFile).PackFileSet(Path.GetDirectoryName(sourceFile), new Dictionary<string, string>
+          {
+            // Note: packedStorageRelativeFile should be in following format: [cc/]aaa.bbb/<hash>/aaa.bb_ 
+            {Path.GetFileName(Path.GetDirectoryName(Path.GetDirectoryName(packedStorageRelativeFile)))!, Path.GetFileName(sourceFile)}
+          });
+
+        await using var stream = File.Open(tempFile, FileMode.Open, FileAccess.ReadWrite);
+        PatchCompressed(File.GetLastWriteTime(sourceFile), stream);
+        await writeStorageFile(packedStorageRelativeFile, stream.Length, stream.Rewind());
+      }
+      finally
+      {
+        File.Delete(tempFile);
+      }
+    }
+
+    private static void PatchCompressed(DateTime writeSourceFileTime, [NotNull] Stream stream)
+    {
+      if (stream == null)
+        throw new ArgumentNullException(nameof(stream));
+      // Bug: The C# library randomizes CCAB::setID in CabPacker::CreateFci(): `pccab.setID = checked ((short) new Random().Next((int) short.MinValue, 32768));`.
+      //      See https://docs.microsoft.com/en-us/windows/win32/api/fci/ns-fci-ccab for details
+
+      var buffer = new byte[0x40];
+
+      var pos = stream.Position;
+      if (stream.Read(buffer, 0, buffer.Length) != buffer.Length)
+        throw new FormatException("Too short Microsoft CAB file");
+
+      if (buffer[0] != 'M' || buffer[1] != 'S' || buffer[2] != 'C' || buffer[3] != 'F')
+        throw new FormatException("Microsoft CAB file is expected");
+
+      var span = TimeSpan.FromSeconds(2);
+      var ceil = writeSourceFileTime.ToCeil(span);
+      var floor = writeSourceFileTime.ToFloor(span);
+      if (floor.Year >= 1980)
+      {
+        var cab = DateTimeUtil.ToDateTime(
+          ToUInt16(buffer[0x37], buffer[0x36]),
+          ToUInt16(buffer[0x39], buffer[0x38]));
+        if (cab < floor || ceil < cab)
+          throw new FormatException("The time in the CAB-file record is out of 2 seconds range which can be patched");
+      }
+      else
+      {
+        throw new FormatException("The source time after rounding to floor is early then 1.1.1980");
+      }
+
+      // setID
+      const ushort id = 0xFFFF;
+      buffer[0x20] = id & 0xFF;
+      buffer[0x21] = id >> 8;
+
+      // DOS date + DOS time, see https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-filetimetodosdatetime
+      var dateTime = new DateTime(1980, 1, 1, 0, 0, 0);
+      var date = dateTime.ToDosDate();
+      var time = dateTime.ToDosTime();
+      buffer[0x36] = (byte) date;
+      buffer[0x37] = (byte) (date >> 8);
+      buffer[0x38] = (byte) time;
+      buffer[0x39] = (byte) (time >> 8);
+
+      stream.Seek(pos, SeekOrigin.Begin);
+      stream.Write(buffer, 0, buffer.Length);
+    }
+
+    private static ushort ToUInt16(byte hi, byte lo)
+    {
+      return (ushort) ((hi << 8) | lo);
+    }
+  }
+}

Core/src/Impl/Commands/DateTimeUtil.cs

@@ -0,0 +1,48 @@
+using System;
+
+namespace JetBrains.SymbolStorage.Impl.Commands
+{
+  internal static class DateTimeUtil
+  {
+    public static ushort ToDosDate(this DateTime date)
+    {
+      if (date.Year < 1980)
+        throw new ApplicationException("The year should be 1980+");
+      return (ushort) (
+        ((date.Year - 1980) << 9) |
+        (date.Month << 5) |
+        date.Day);
+    }
+
+    public static ushort ToDosTime(this DateTime time)
+    {
+      return (ushort) (
+        (time.Hour << 11) |
+        (time.Minute << 5) |
+        (time.Second / 2));
+    }
+
+    public static DateTime ToDateTime(ushort date, ushort time)
+    {
+      return new DateTime(
+        (date >> 9) + 1980,
+        (date >> 5) & 0xF,
+        date & 0x1F,
+        time >> 11,
+        (time >> 5) & 0x3F,
+        (time & 0x1F) << 1);
+    }
+
+    public static DateTime ToFloor(this DateTime date, TimeSpan span)
+    {
+      var ticks = date.Ticks / span.Ticks;
+      return new DateTime(ticks * span.Ticks);
+    }
+
+    public static DateTime ToCeil(this DateTime date, TimeSpan span)
+    {
+      var ticks = (date.Ticks + span.Ticks - 1) / span.Ticks;
+      return new DateTime(ticks * span.Ticks);
+    }
+  }
+}

Core/src/Impl/Commands/DeleteCommand.cs

@@ -0,0 +1,69 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using JetBrains.Annotations;
+using JetBrains.SymbolStorage.Impl.Logger;
+using JetBrains.SymbolStorage.Impl.Storages;
+
+namespace JetBrains.SymbolStorage.Impl.Commands
+{
+  internal sealed class DeleteCommand
+  {
+    private readonly ILogger myLogger;
+    private readonly IStorage myStorage;
+    private readonly IReadOnlyCollection<string> myIncProductWildcards;
+    private readonly IReadOnlyCollection<string> myExcProductWildcards;
+    private readonly IReadOnlyCollection<string> myIncVersionWildcards;
+    private readonly IReadOnlyCollection<string> myExcVersionWildcards;
+
+    public DeleteCommand(
+      [NotNull] ILogger logger,
+      [NotNull] IStorage storage,
+      [NotNull] IReadOnlyCollection<string> incProductWildcards,
+      [NotNull] IReadOnlyCollection<string> excProductWildcards,
+      [NotNull] IReadOnlyCollection<string> incVersionWildcards,
+      [NotNull] IReadOnlyCollection<string> excVersionWildcards)
+    {
+      myLogger = logger ?? throw new ArgumentNullException(nameof(logger));
+      myStorage = storage ?? throw new ArgumentNullException(nameof(storage));
+      myIncProductWildcards = incProductWildcards ?? throw new ArgumentNullException(nameof(incProductWildcards));
+      myExcProductWildcards = excProductWildcards ?? throw new ArgumentNullException(nameof(excProductWildcards));
+      myIncVersionWildcards = incVersionWildcards ?? throw new ArgumentNullException(nameof(incVersionWildcards));
+      myExcVersionWildcards = excVersionWildcards ?? throw new ArgumentNullException(nameof(excVersionWildcards));
+    }
+
+    public async Task<int> Execute()
+    {
+      var validator = new Validator(myLogger, myStorage);
+      var storageFormat = await validator.ValidateStorageMarkers();
+
+      long deleteTags;
+      {
+        var tagItems = await validator.LoadTagItems(
+          myIncProductWildcards,
+          myExcProductWildcards,
+          myIncVersionWildcards,
+          myExcVersionWildcards);
+        validator.DumpProducts(tagItems);
+        validator.DumpProperties(tagItems);
+        deleteTags = tagItems.Count;
+
+        myLogger.Info($"[{DateTime.Now:s}] Deleting tag files...");
+        foreach (var tagItem in tagItems)
+        {
+          var file = tagItem.Key;
+          myLogger.Info($"  Deleting {file}");
+          await myStorage.Delete(file);
+        }
+      }
+
+      {
+        var tagItems = await validator.LoadTagItems();
+        var (_, files) = await validator.GatherDataFiles();
+        var (statistics, deleted) = await validator.Validate(tagItems, files, storageFormat, Validator.ValidateMode.Delete);
+        myLogger.Info($"[{DateTime.Now:s}] Done (deleted tag files: {deleteTags}, deleted data files: {deleted}, warnings: {statistics.Warnings}, errors: {statistics.Errors}, fixes: {statistics.Fixes})");
+        return statistics.HasProblems ? 1 : 0;
+      }
+    }
+  }
+}

Core/src/Impl/Commands/KeyType.cs

@@ -0,0 +1,10 @@
+namespace JetBrains.SymbolStorage.Impl.Commands
+{
+  internal enum KeyType
+  {
+    Other = 0,
+    WPdb,
+    Pe,
+    Elf
+  }
+}