Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update existing repo: #122

Closed
wants to merge 2 commits into from
Closed

Update existing repo: #122

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f5c79bb
Update existing repo feature:
tommasobertoni Mar 12, 2017
03d4b3a
Removed NoSolutionsFoundException, added ProcessRepoResult enum.
tommasobertoni Mar 25, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions src/SourceBrowser.Generator/SolutionAnalyzer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,21 @@ namespace SourceBrowser.Generator
{
public class SolutionAnalayzer
{
private static object _sync = new object();

MSBuildWorkspace _workspace;
Solution _solution;
private string _solutionPath;
private ReferencesourceLinkProvider _refsourceLinkProvider = new ReferencesourceLinkProvider();

public SolutionAnalayzer(string solutionPath)
{
_workspace = MSBuildWorkspace.Create();
lock (_sync)
{
_workspace = MSBuildWorkspace.Create();
}

_solutionPath = solutionPath;
_workspace.WorkspaceFailed += _workspace_WorkspaceFailed;
_solution = _workspace.OpenSolutionAsync(solutionPath).Result;
_refsourceLinkProvider.Init();
Expand All @@ -49,7 +57,9 @@ private void _workspace_WorkspaceFailed(object sender, WorkspaceDiagnosticEventA

if (!Directory.Exists(logDirectory))
Directory.CreateDirectory(logDirectory);
var logPath = logDirectory + "log.txt";

var logName = Path.GetFileName(_solutionPath);
var logPath = $"{logDirectory}{logName}.log.txt";
using (var sw = new StreamWriter(logPath))
{
sw.Write(e);
Expand Down
4 changes: 2 additions & 2 deletions src/SourceBrowser.Generator/Transformers/AbstractWorkspaceVisitor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public virtual void Visit(WorkspaceModel workspaceModel)

protected virtual void VisitWorkspace(WorkspaceModel workspaceModel)
{
foreach(var child in _workspaceModel.Children)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Weird, we're not even using this backing field _workspaceModel.

foreach(var child in workspaceModel.Children)
{
VisitProjectItem(child);
}
Expand Down Expand Up @@ -54,7 +54,7 @@ protected virtual void VisitProjectItem(IProjectItem projectItem)

protected virtual void VisitFolder(FolderModel folderModel)
{
foreach(var child in folderModel.Children)
foreach(var child in folderModel.Children.OrderBy(proj => proj.Name))
{
VisitProjectItem(child);
}
Expand Down
48 changes: 42 additions & 6 deletions src/SourceBrowser.Generator/Transformers/TreeViewTransformer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ namespace SourceBrowser.Generator.Transformers
public class TreeViewTransformer : AbstractWorkspaceVisitor
{
private string _savePath;
HtmlTextWriter _writer;
private StreamWriter _sw;
private HtmlTextWriter _writer;
private readonly string _userNameAndRepoPrefix;

private const string _treeViewOutputFile = "treeView.html";
Expand All @@ -34,20 +35,55 @@ public TreeViewTransformer(string savePath, string userName, string repoName)

protected override void VisitWorkspace(WorkspaceModel workspaceModel)
{
using (var stringWriter = new StreamWriter(_savePath, false))
using(_writer = new HtmlTextWriter(stringWriter))
// The first WorkspaceModel that is visited is the root of the tree view
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand why this code had to change. This PR is making it possible to re-analyze solutions so I wouldn't think this would be a required change.

// and its children are the solutions.

bool disposeWriters = false;

// Create the writers only if they're null
if (_sw == null)
{
_sw = new StreamWriter(_savePath, false);

if (_writer != null)
_writer.Dispose();

_writer = new HtmlTextWriter(_sw);

disposeWriters = true;
}

if (disposeWriters)
{
// The current WorkspaceModel is the root node, no need to increase the depth
_writer.AddAttribute(HtmlTextWriterAttribute.Id, "browserTree");
_writer.AddAttribute(HtmlTextWriterAttribute.Class, "treeview");
_writer.AddAttribute("data-role", "treeview");
_writer.RenderBeginTag(HtmlTextWriterTag.Ul);

}
else
{
// The current WorkspaceModel is a Child of the root node.
depth++;
base.VisitWorkspace(workspaceModel);
depth--;
}

base.VisitWorkspace(workspaceModel);

if (disposeWriters)
{
// The current WorkspaceModel is the root node.
// Every child has been visited: dispose the writers.

disposeWriters = false;
_writer.RenderEndTag();
_writer.WriteLine();
_writer.Dispose();
_sw.Dispose();
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

_sw should be null so that it's created again. But I also don't understand why this change is necessary.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll address here your doubts about the changes in the TreeViewTransformer.VisitWorkspace method:

the changes that I made to make the application able to process multiple solutions in the same repository were to process all the .sln files in parallel and when this processing was finished a new WorkspaceModel item was created and it's children were the WorkspaceModel resulted from the solutions.

When the TreeViewTransformer visited the first WorkspaceModel (which is the root) it created the writers and then it invoked the base method, which recursively visited all the sub nodes.
This means that when the transformer visited the solutions' WorkspaceModels under the root, it created again the writers (because projectItem is a WorkspaceModel), and therefore tried to create the same output file, which lead to an IOException: "The process cannot access the file 'treeView.html' because it is being used by another process."

Anyway, what I wrote was more of a hack than a proper fix, so I'd like to use another approach: by restoring the TreeViewTransformer class as it was before of my changes, we can bypass the same problem by adding all the solutions' children to the root node, instead of the solutions' WorkspaceModels.

So, in the UploadRepository.ProcessRepo method, replace

// Add all the results to the root workspace model.
foreach (var workspace in processedWorkspaces)
    rootWorkspaceModel.Children.Add(workspace);

// WorkspaceModel (root)
// - WorkspaceModel (sln1)
//   - FolderModel
//     - ...
// - WorkspaceModel (sln2)
//   - FolderModel
//     - ...

with

foreach (var children in processedWorkspaces.SelectMany(e => e.Children))
    rootWorkspaceModel.Children.Add(children);

// WorkspaceModel (root)
// - FolderModel (from sln1's WorkspaceModel)
//   - ...
// - FolderModel (from sln2's WorkspaceModel)
//   - ...

resulting in a single WorkspaceModel.

Please, give me your opinions about this. Thanks!

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi,
I tried in implementing the changes that I described above, but in some cases there are problems with repositories with many solutions such that the result is not really good.
Take for example Hangfire: there are two solutions in the root and they share some source files. By flattening the hierarchy in the root WorkplaceModel, the generated treeview is not really readable

sb_hang_old

I tried to solve this by adding the solutions' children to a new FolderModel instead of directly to the root, and indeed the new treeview was better

sb_hang_new

but then the navigation didn't work well because the relative paths didn't match the files in the treeview anymore.

I'm thinking that this could be solved by navigating to a solution's treeview when you click on the solution's name, so basically when you go to /Browse/{user}/{repo} the treeview section shows the available solutions.

But before going this path, I need your opinions.
Thank you

}
else
{
// The current WorkspaceModel is a Child of the root node.
depth--;
}
}

Expand Down
13 changes: 10 additions & 3 deletions src/SourceBrowser.Site/Controllers/BrowseController.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,10 +111,17 @@ private string loadTreeView(string username, string repository)
var organizationPath = System.Web.Hosting.HostingEnvironment.MapPath("~/") + "SB_Files\\";
string treeViewFileName = "treeview.html";
var treeViewPath = Path.Combine(organizationPath, username, repository, treeViewFileName);

using (var treeViewFile = new StreamReader(treeViewPath))

try
{
using (var treeViewFile = new StreamReader(treeViewPath))
{
return treeViewFile.ReadToEnd();
}
}
catch (FileNotFoundException)
{
return treeViewFile.ReadToEnd();
return $"{treeViewFileName} was not found";
}
}

Expand Down
187 changes: 129 additions & 58 deletions src/SourceBrowser.Site/Controllers/UploadController.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using SourceBrowser.Site.Repositories;
using System;
using System.Linq;
using System.Web.Routing;

public class UploadController : Controller
{
Expand Down Expand Up @@ -34,18 +35,25 @@ public ActionResult Submit(string githubUrl)
// Check if this repo already exists
if (!BrowserRepository.TryLockRepository(retriever.UserName, retriever.RepoName))
{
// Repo exists. Redirect the user to that repository.
return Redirect("/Browse/" + retriever.UserName + "/" + retriever.RepoName);
// Repo exists. Redirect the user to the Update view
// where he can choose if he wants to visit the existing repo or update it.
var routeValues = new RouteValueDictionary();
routeValues.Add(nameof(githubUrl), githubUrl);
return RedirectToAction("Update", routeValues);
}

// We have locked the repository and marked it as processing.
// Whenever we return or exit on an exception, we need to unlock this repository
bool processingSuccessful = false;

ProcessRepoResult result = ProcessRepoResult.Failed;

try
{
string repoRootPath = string.Empty;
string repoSourceStagingPath = null;

try
{
repoRootPath = retriever.RetrieveProject();
repoSourceStagingPath = retriever.RetrieveProject();
}
catch (Exception ex)
{
Expand All @@ -54,64 +62,32 @@ public ActionResult Submit(string githubUrl)
}

// Generate the source browser files for this solution
var solutionPaths = GetSolutionPaths(repoRootPath);
if (solutionPaths.Length == 0)
{
ViewBag.Error = "No C# solution was found. Ensure that a valid .sln file exists within your repository.";
return View("Index");
}

var organizationPath = System.Web.Hosting.HostingEnvironment.MapPath("~/") + "SB_Files\\" + retriever.UserName;
var repoPath = Path.Combine(organizationPath, retriever.RepoName);

// TODO: Use parallel for.
// TODO: Process all solutions.
// For now, we're assuming the shallowest and shortest .sln file is the one we're interested in
foreach (var solutionPath in solutionPaths.OrderBy(n => n.Length).Take(1))
{
try
{
var workspaceModel = UploadRepository.ProcessSolution(solutionPath, repoRootPath);

//One pass to lookup all declarations
var typeTransformer = new TokenLookupTransformer();
typeTransformer.Visit(workspaceModel);
var tokenLookup = typeTransformer.TokenLookup;

//Another pass to generate HTMLs
var htmlTransformer = new HtmlTransformer(tokenLookup, repoPath);
htmlTransformer.Visit(workspaceModel);

var searchTransformer = new SearchIndexTransformer(retriever.UserName, retriever.RepoName);
searchTransformer.Visit(workspaceModel);

// Generate HTML of the tree view
var treeViewTransformer = new TreeViewTransformer(repoPath, retriever.UserName, retriever.RepoName);
treeViewTransformer.Visit(workspaceModel);
}
catch (Exception ex)
{
// TODO: Log this
ViewBag.Error = "There was an error processing solution " + Path.GetFileName(solutionPath);
return View("Index");
}
}
var parsedRepoPath = Path.Combine(organizationPath, retriever.RepoName);

try
{
UploadRepository.SaveReadme(repoPath, retriever.ProvideParsedReadme());
result = UploadRepository.ProcessRepo(retriever, repoSourceStagingPath, parsedRepoPath);
}
catch (Exception ex)
{
// TODO: Log and swallow - readme is not essential.
// TODO: Log this
ViewBag.Error = "There was an error processing solution."/* + Path.GetFileName(solutionPath)*/;
return View("Index");
}

if (result == ProcessRepoResult.NoSolutionsFound)
{
ViewBag.Error = "No C# solution was found. Ensure that a valid .sln file exists within your repository.";
return View("Index");
}

processingSuccessful = true;
return Redirect("/Browse/" + retriever.UserName + "/" + retriever.RepoName);
}
finally
{
if (processingSuccessful)
if (result == ProcessRepoResult.Successful)
{
BrowserRepository.UnlockRepository(retriever.UserName, retriever.RepoName);
}
Expand All @@ -123,17 +99,112 @@ public ActionResult Submit(string githubUrl)
}

/// <summary>
/// Simply searches for the solution files and returns their paths.
/// This API is used for the updating an existing repo.
/// If the request's http method is GET, it will return a View with a link to the existing repo
/// and a form that allows the user to force the update.
/// If the request's http method is POST, it will delete the existing repo and then
/// it will process the github url.
/// </summary>
/// <param name="rootDirectory">
/// The root Directory.
/// </param>
/// <returns>
/// The solution paths.
/// </returns>
private string[] GetSolutionPaths(string rootDirectory)
/// <param name="githubUrl">The github url of the repo.</param>
[AcceptVerbs(HttpVerbs.Get | HttpVerbs.Post)]
public ActionResult Update(string githubUrl)
{
return Directory.GetFiles(rootDirectory, "*.sln", SearchOption.AllDirectories);
// If someone navigates to submit directly, just send 'em back to index
if (string.IsNullOrWhiteSpace(githubUrl))
{
return View("Index");
}

var retriever = new GitHubRetriever(githubUrl);
if (!retriever.IsValidUrl())
{
ViewBag.Error = "Make sure that the provided path points to a valid GitHub repository.";
return View("Index");
}

// Check if this repo already exists
if (!BrowserRepository.PathExists(retriever.UserName, retriever.RepoName))
{
// Repo doesn't exist.
ViewBag.Error = $"The repo \"{githubUrl}\" cannot be updated because it was never submitted.";
return View("Index");
}

string currentHttpMethod = HttpContext.Request.HttpMethod.ToUpper();

if (currentHttpMethod == HttpVerbs.Get.ToString().ToUpper())
{
ViewBag.BrowseUrl = "/Browse/" + retriever.UserName + "/" + retriever.RepoName;
ViewBag.GithubUrl = githubUrl;
return View("Update");
}
else if (currentHttpMethod == HttpVerbs.Post.ToString().ToUpper())
{
// Remove old files
BrowserRepository.RemoveRepository(retriever.UserName, retriever.RepoName);
// Create a file that indicates that the upload will begin
BrowserRepository.TryLockRepository(retriever.UserName, retriever.RepoName);

// We have locked the repository and marked it as processing.
// Whenever we return or exit on an exception, we need to unlock this repository

ProcessRepoResult result = ProcessRepoResult.Failed;

try
{
string repoSourceStagingPath = null;

try
{
repoSourceStagingPath = retriever.RetrieveProject();
}
catch (Exception ex)
{
ViewBag.Error = "There was an error downloading this repository.";
return View("Index");
}

var organizationPath = System.Web.Hosting.HostingEnvironment.MapPath("~/") + "SB_Files\\" + retriever.UserName;
var parsedRepoPath = Path.Combine(organizationPath, retriever.RepoName);

try
{
// Generate the source browser files for this solution
result = UploadRepository.ProcessRepo(retriever, repoSourceStagingPath, parsedRepoPath);
}
catch (Exception ex)
{
// TODO: Log this
ViewBag.Error = "There was an error processing solution."/* + Path.GetFileName(solutionPath)*/;
return View("Index");
}

if (result == ProcessRepoResult.NoSolutionsFound)
{
ViewBag.Error = "No C# solution was found. Ensure that a valid .sln file exists within your repository.";
return View("Index");
}

return Redirect("/Browse/" + retriever.UserName + "/" + retriever.RepoName);
}
finally
{
if (result == ProcessRepoResult.Successful)
{
BrowserRepository.UnlockRepository(retriever.UserName, retriever.RepoName);
}
else
{
BrowserRepository.RemoveRepository(retriever.UserName, retriever.RepoName);
}
}
}
else
{
// The request's http method wasn't valid: back to index
ViewBag.Error = $"Bad request.";
return View("Index");
}
}
}
}
Loading