fixing chunk calculator

.github/workflows/build.yml

@@ -1,7 +1,7 @@
 name: build
 
 env:
-  v: '1.4.0'
+  v: '1.5.0'
   av: '1.0.0'
 
 on:

docs/release-history.md

@@ -10,6 +10,7 @@
 - S3 `Ls` operation was returning it's own folder object in the list.
 - S3 `Ls` was facing number parsing overflow on multi-gigabyte files.
 - On some platforms `hmac256` was not working resulting in failure to sign AWS requests.
+- Multi-part uploads in S3 were sometimes not calculating chunk sizes properly.
 
 ## 1.4.0
 

src/Stowage.Terminal/AppTopLevel.cs

@@ -51,13 +51,14 @@ public AppTopLevel(IFileStorage fs) {
                 new StatusItem(Key.F4, "~F4~ Edit", () => {
                     GetFocusedView().ViewEntry();
                 }),
-                new StatusItem(Key.F5, "~F5~ Copy", () => {
-                    CopyFiles();
-
+                new StatusItem(Key.F5, "~F5~ Copy", CopyFiles),
+                new StatusItem(Key.R | Key.CtrlMask, "~Ctrl-R~ Rescan", () => {
+                    GetFocusedView().Ls();
                 }),
-                new StatusItem(Key.F10, "~F10~ Quit", () => {
-                    RequestStop();
-                })
+                new StatusItem(Key.Delete, "~Del~ Delete", () => {
+                    GetFocusedView().DeleteEntry();
+                }),
+                new StatusItem(Key.F10, "~F10~ Quit", RequestStop)
 
             };
 
@@ -106,7 +107,7 @@ private FSView GetUnfocusedView() {
             if(_fsView1.HasFocus)
                 return _fsView2;
 
-            return _fsView2;
+            return _fsView1;
         }
 
 

src/Stowage.Terminal/FSView.cs

@@ -3,17 +3,13 @@
 
 namespace Stowage.Terminal {
     class FSView : View {
-        private readonly IFileStorage _fs;
-        private IOPath _currentPath;
         private readonly Label _pathLabel;
         private readonly ListView _entryList = new ListView();
-        private IOEntry? _activeEntry = null;
 
         public event Action<IFileStorage, IOEntry> SelectedEntryChanged;
-
         public FSView(IFileStorage fs, IOPath? startPath = null) {
-            _fs = fs;
-            _currentPath = startPath ?? IOPath.Root;
+            Fs = fs;
+            CurrentPath = startPath ?? IOPath.Root;
             _pathLabel = new Label() { X = 0, Y = 0 };
 
             Add(new Label("Name") { X = 2, Y = 1, ColorScheme = Colors.Menu });
@@ -39,13 +35,17 @@ public FSView(IFileStorage fs, IOPath? startPath = null) {
             _entryList.SelectedItemChanged += _entryList_SelectedItemChanged;
         }
 
+        public IFileStorage Fs { get; init; }
+        public IOEntry? SelectedEntry { get; private set; }
+        public IOPath CurrentPath { get; private set; }
+
         private void _entryList_SelectedItemChanged(ListViewItemEventArgs args) {
             if(args.Value is not IOEntry entry)
                 return;
 
-            _activeEntry = entry;
+            SelectedEntry = entry;
 
-            SelectedEntryChanged?.Invoke(_fs, entry);
+            SelectedEntryChanged?.Invoke(Fs, entry);
         }
 
         private void _entryList_OpenSelectedItem(ListViewItemEventArgs args) {
@@ -55,23 +55,23 @@ private void _entryList_OpenSelectedItem(ListViewItemEventArgs args) {
             if(entry.Path.IsFolder) {
                 // enter the folder
                 if(entry.Name == "../")
-                    _currentPath = _currentPath.Parent;
+                    CurrentPath = CurrentPath.Parent;
                 else
-                    _currentPath = entry.Path;
+                    CurrentPath = entry.Path;
                 Ls();
             } else {
 
             }
         }
 
-        private void Ls() {
+        public void Ls() {
             Task.Run(async () => {
                 try {
-                    _pathLabel.Text = _currentPath.Full;
-                    IReadOnlyCollection<IOEntry> entries = await _fs.Ls(_currentPath);
+                    _pathLabel.Text = CurrentPath.Full;
+                    IReadOnlyCollection<IOEntry> entries = await Fs.Ls(CurrentPath);
 
                     Application.MainLoop.Invoke(() => {
-                        _entryList.Source = new IOEntryListDataSource(entries, _currentPath.IsRootPath);
+                        _entryList.Source = new IOEntryListDataSource(entries, CurrentPath.IsRootPath);
                     });
                 } catch(Exception ex) {
                     MessageBox.ErrorQuery(60, 10, "Error", ex.ToString(), "Ok");
@@ -125,7 +125,7 @@ private void AddProperties(View view, int firstColWidth, params string[] args) {
 
         public void ShowEntryDetails() {
 
-            if(_activeEntry == null)
+            if(SelectedEntry == null)
                 return;
 
             var close = new Button("Close", true);
@@ -135,15 +135,15 @@ public void ShowEntryDetails() {
 
             var dialog = new Dialog("Entry Details", 70, 20, close);
             var props = new List<string>{
-                "name", _activeEntry.Name,
-                "size", _activeEntry.Size == null ? "" : _activeEntry.Size.Value.Bytes().Humanize(),
-                "created", _activeEntry.CreatedTime == null ? "" : _activeEntry.CreatedTime.Value.ToString(),
-                "modified", _activeEntry.LastModificationTime == null ? "" : _activeEntry.LastModificationTime.Value.ToString(),
-                "MD-5", _activeEntry.MD5
+                "name", SelectedEntry.Name,
+                "size", SelectedEntry.Size == null ? "" : SelectedEntry.Size.Value.Bytes().Humanize(),
+                "created", SelectedEntry.CreatedTime == null ? "" : SelectedEntry.CreatedTime.Value.ToString(),
+                "modified", SelectedEntry.LastModificationTime == null ? "" : SelectedEntry.LastModificationTime.Value.ToString(),
+                "MD-5", SelectedEntry.MD5
             };
 
-            if(_activeEntry.Properties != null)
-                props.AddRange(_activeEntry.Properties.SelectMany(p => new[] { p.Key, p.Value?.ToString() }));
+            if(SelectedEntry.Properties != null)
+                props.AddRange(SelectedEntry.Properties.SelectMany(p => new[] { p.Key, p.Value?.ToString() }));
 
             // basic properties
             AddProperties(dialog, 20, props.ToArray());
@@ -152,39 +152,33 @@ public void ShowEntryDetails() {
         }
 
         public void ViewEntry() {
-            if(_activeEntry == null)
+            if(SelectedEntry == null)
                 return;
 
-            var editor = new TextFileEditorWindow(_fs, _activeEntry);
+            var editor = new TextFileEditorWindow(Fs, SelectedEntry);
             Application.Run(editor);
+        }
+
+        public void DeleteEntry() {
+            if(SelectedEntry == null)
+                return;
+
+            if(0 == MessageBox.Query("Delete", $"Delete '{SelectedEntry.Path.Full}'?", "Yes", "No")) {
+                Task.Run(async () => {
+                    try {
+                        await Fs.Rm(SelectedEntry.Path);
 
+                        Application.MainLoop.Invoke(() => {
+                            MessageBox.Query("Deleted", "Object(s) deleted.", "Ok");
+                        });
 
-            //var viewer = new Dialog(_activeEntry.Name) {
-            //    X = 2,
-            //    Y = 2,
-            //    Width = Dim.Fill() - 4,
-            //    Height = Dim.Fill() - 4
-            //};
-            //var label = new Label("Loading...") {
-            //    X = 0,
-            //    Y = 0,
-            //    Width = Dim.Fill(),
-            //    Height = Dim.Fill()
-            //};
-            //viewer.Add(label);
-
-            //Task.Run(async () => {
-            //    try {
-            //        string? content = await _fs.ReadText(_activeEntry.Path);
-            //        Application.MainLoop.Invoke(() => {
-            //            label.Text = content;
-            //        });
-            //    } catch(Exception ex) {
-            //        Console.WriteLine(ex.ToString());
-            //    }
-            //});
-
-            //Application.Run(viewer);
+                        Ls();
+                    } catch(Exception ex) {
+                        MessageBox.ErrorQuery(60, 10, "Error", ex.ToString(), "Ok");
+                    }
+                });
+            }
         }
+
     }
 }

src/Stowage.Terminal/FileCopyDialog.cs

@@ -1,27 +1,43 @@
 using System;
+using System.Buffers;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.Threading;
 using System.Threading.Tasks;
+using Humanizer;
 using Terminal.Gui;
 
 namespace Stowage.Terminal {
     class FileCopyDialog {
+
+        private const int DefaultCopyBufferSize = 81920;
+
         private readonly View _parent;
+        private readonly CancellationTokenSource _cts = new CancellationTokenSource();
         private readonly FSView _from;
         private readonly FSView _to;
 
         private readonly Dialog _dialog;
         private readonly ProgressBar _progressTotal;
         private readonly ProgressBar _progressCurrent;
+        private readonly Label _labelTotal;
+        private readonly Label _labelCurrent;
+        private long _copiedTotal = 0;
+        private long _sizeTotal = 0;
 
         public FileCopyDialog(View parent, FSView from, FSView to) {
             _parent = parent;
             _from = from;
             _to = to;
 
             _parent.GetCurrentWidth(out int width);
-            _dialog = new Dialog("Copy files", width / 2, 10);
+            var cancel = new Button("Cancel");
+            cancel.Clicked += () => {
+                _cts.Cancel();
+            };
+
+            _dialog = new Dialog("Copy files", width / 2, 10, cancel);
 
             var lbl = new Label("  total: ") { X = 0, Y = 0 };
             _dialog.Add(lbl);
@@ -34,7 +50,10 @@ public FileCopyDialog(View parent, FSView from, FSView to) {
             };
             _dialog.Add(_progressTotal);
 
-            lbl = new Label("current: ") { X = 0, Y = Pos.Bottom(lbl) };
+            _labelTotal = new Label("0 / 0") { X = 0, Y = Pos.Bottom(lbl) };
+            _dialog.Add(_labelTotal);
+
+            lbl = new Label("current: ") { X = 0, Y = Pos.Bottom(_labelTotal) };
             _dialog.Add(lbl);
             _progressCurrent = new ProgressBar() {
                 X = Pos.Right(lbl),
@@ -43,14 +62,91 @@ public FileCopyDialog(View parent, FSView from, FSView to) {
                 Height = 1
             };
             _dialog.Add(_progressCurrent);
+
+            _labelCurrent = new Label("0 / 0") { X = 0, Y = Pos.Bottom(lbl) };
+            _dialog.Add(_labelCurrent);
         }
 
         public void Start() {
 
-            _progressCurrent.Fraction = 0.2f;
-            _progressTotal.Fraction = 0.5f;
+            RunCopy();
 
             Application.Run(_dialog);
         }
+
+        private async Task<IReadOnlyCollection<IOEntry>> Explode(IFileStorage fs, IOEntry entry) {
+            var result = new List<IOEntry>();
+
+            if(entry.Path.IsFile) {
+                result.Add(entry);
+            } else {
+                IReadOnlyCollection<IOEntry> entries = await fs.Ls(entry.Path, true, _cts.Token);
+                result.AddRange(entries.Where(e => e.Path.IsFile));
+            }
+
+            return result;
+        }
+
+        private async Task Copy(IFileStorage fsFrom, IFileStorage fsTo, IOPath pathFrom, IOPath pathTo) {
+            _progressCurrent.Fraction = 0.0f;
+
+            await using(Stream? streamFrom = await fsFrom.OpenRead(pathFrom, _cts.Token)) {
+                await using(Stream streamTo = await fsTo.OpenWrite(pathTo, _cts.Token)) {
+                    if(streamFrom == null || streamTo == null)
+                        return;
+
+                    byte[] buffer = ArrayPool<byte>.Shared.Rent(DefaultCopyBufferSize);
+                    try {
+                        int bytesRead;
+                        while((bytesRead = await streamFrom.ReadAsync(new Memory<byte>(buffer), _cts.Token)) != 0) {
+                            await streamTo.WriteAsync(new ReadOnlyMemory<byte>(buffer, 0, bytesRead), _cts.Token);
+                            float fraction = streamFrom.Position / (float)streamFrom.Length;
+
+                            Application.MainLoop.Invoke(() => {
+                                _progressCurrent.Fraction = fraction;
+                                _copiedTotal += bytesRead;
+                                _labelCurrent.Text = $"{streamFrom.Position.Bytes()} / {streamFrom.Length.Bytes()}";
+                                _labelTotal.Text = $"{_copiedTotal.Bytes()} / {_sizeTotal.Bytes()}";
+                            });
+                        }
+
+                    } finally {
+                        ArrayPool<byte>.Shared.Return(buffer);
+                    }
+                }
+            }
+        }
+
+        private void RunCopy() {
+            _progressCurrent.Fraction = 0.0f;
+            _progressTotal.Fraction = 0.0f;
+
+            Task.Run(async () => {
+                Exception? ex = null;
+                try {
+                    IReadOnlyCollection<IOEntry> sourceEntries = await Explode(_from.Fs, _from.SelectedEntry!);
+                    _sizeTotal = sourceEntries.Sum(e => e.Size!.Value);
+
+                    int i = 0;
+                    foreach(IOEntry entry in sourceEntries) {
+
+                        await Copy(_from.Fs, _to.Fs, entry.Path, _to.CurrentPath.Combine(entry.Path.Name));
+
+                        Application.MainLoop.Invoke(() => {
+                            _progressTotal.Fraction = i++ * 100.0f / sourceEntries.Count;
+                        });
+                    }
+                } catch(Exception ex1) {
+                    ex = ex1;
+                }   
+
+                Application.MainLoop.Invoke(() => {
+                    if(ex != null) {
+                        MessageBox.ErrorQuery(60, 10, "Error", ex.ToString(), "Ok");
+                    }
+                    Application.RequestStop();
+                });
+            });
+        }
     }
 }