ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: vs plugin cdb writing

Browse Source 
cdb data will now be written to file immediatly instead of stored and then written all at once
this might solve memory issues for very large projects, but also increases parsing time (added concurency)
This commit is contained in:
wrongway88
2017-02-23 00:01:18 +01:00
parent 4563cbe90a
commit e6f7e61a5f
7 changed files with 202 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ide_plugins/vs/coati_plugin_vs.vsix
BIN
View File
Binary file not shown.
ide_plugins/vs/vs2015/CoatiPlugin/CoatiPlugin/CoatiPlugin.csproj
+1
View File
@@ -162,6 +162,7 @@
<Compile Include="SolutionParser\CompilationDatabase.cs" />
<Compile Include="SolutionParser\SolutionParser.cs" />
<Compile Include="Utility\ProjectUtility.cs" />
<Compile Include="Utility\QueuedFileWriter.cs" />
<Compile Include="Utility\SolutionUtility.cs" />
<Compile Include="Utility\SystemUtility.cs" />
<Compile Include="Utility\StringUtility.cs" />
ide_plugins/vs/vs2015/CoatiPlugin/CoatiPlugin/CoatiPluginPackage.cs
+4 -61
View File
@@ -366,35 +366,11 @@ namespace CoatiSoftware.CoatiPlugin
_cdbList.AppendOrUpdate(creationResult._cdb);
_cdbList.SaveMetaData();
// string metaData = creationResult._cdb.SerializeMetaDataXML();
Wizard.WindowCDBReady dialog = new Wizard.WindowCDBReady();
dialog.setData(creationResult);
dialog.ShowDialog();
// Utility.DataUtility.GetInstance().AppendData(metaData);
if (WriteCDBToFile(creationResult._cdb, creationResult._cdbDirectory, creationResult._cdbName))
{
Wizard.WindowCDBReady dialog = new Wizard.WindowCDBReady();
dialog.setData(creationResult);
dialog.ShowDialog();
_cdbList.Refresh();
}
else
{
string title = "Error";
string text = "Failed to write CDB '" + creationResult._cdbName + "' to directory \"" + creationResult._cdbDirectory + "\"\n";
if (LoggingEnabled)
{
text += "See log for details.";
}
else
{
text += "Consider enabling logging to provide additional information (Tools->Options->Coati).";
}
DialogResult result = MessageBox.Show(text, title, MessageBoxButtons.OK, MessageBoxIcon.Warning);
}
_cdbList.Refresh();
}
else
{
@@ -402,39 +378,6 @@ namespace CoatiSoftware.CoatiPlugin
}
}
private bool WriteCDBToFile(SolutionParser.CompilationDatabase cdb, string directory, string fileName)
{
try
{
Logging.Logging.LogInfo("Serializing " + cdb.CommandObjectCount + " command objects.");
System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch();
watch.Start();
string content = cdb.SerializeJSON();
watch.Stop();
Logging.Logging.LogInfo("Done serializing: " + watch.ElapsedMilliseconds + " ms");
File.WriteAllText(directory + "\\" + fileName + ".json", content);
}
catch(Exception e)
{
string text = "Failed to write CDB '" + Logging.Obfuscation.NameObfuscator.GetObfuscatedName(fileName) + "' to directory \"" + Logging.Obfuscation.NameObfuscator.GetObfuscatedName(directory) + "\"\n";
Logging.Logging.LogError(text);
Logging.Logging.LogError("Exception: " + e.Message);
_cdbList.UnloadCDBs();
return false;
}
string message = "The CDB '" + Logging.Obfuscation.NameObfuscator.GetObfuscatedName(fileName) + "' was created at directory \"" + Logging.Obfuscation.NameObfuscator.GetObfuscatedName(directory) + "\"\n";
Logging.Logging.LogInfo(message);
_cdbList.UnloadCDBs();
return true;
}
private void MenuItemCallback(object sender, EventArgs e)
{
MenuCommand menuCommand = sender as MenuCommand;
ide_plugins/vs/vs2015/CoatiPlugin/CoatiPlugin/SolutionParser/CompilationDatabase.cs
+4 -6
View File
@@ -2,6 +2,9 @@
using System.Xml.Serialization;
using System.Xml;
using Newtonsoft.Json.Linq;
using System.IO;
using System.Threading;
using System;
namespace CoatiSoftware.CoatiPlugin.SolutionParser
{
@@ -67,14 +70,9 @@ namespace CoatiSoftware.CoatiPlugin.SolutionParser
get { return _commandObjects.Count; }
}
public void AddOrUpdateCommandObject(CommandObject commandObject)
public void AddOrUpdateCommandObject(CommandObject commandObject, bool saveImmediatly = false)
{
_commandObjects.Add(commandObject); // updating is not efficient as it is
//if (TryUpdateCommandObject(commandObject) == false)
//{
// _commandObjects.Add(commandObject);
//}
}
// remove commandObjects for removed files
ide_plugins/vs/vs2015/CoatiPlugin/CoatiPlugin/Utility/QueuedFileWriter.cs
+158
View File
@@ -0,0 +1,158 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
namespace CoatiSoftware.CoatiPlugin.Utility
{
class QueuedFileWriter
{
private bool _working = false;
Thread _workerThread = null;
private static ReaderWriterLockSlim _statusLock = new ReaderWriterLockSlim();
private static ReaderWriterLockSlim _queueLock = new ReaderWriterLockSlim();
private static ReaderWriterLockSlim _fileLock = new ReaderWriterLockSlim();
private Queue<string> _inputQueue = new Queue<string>();
private Queue<string> _outputQueue = new Queue<string>();
private string _targetDirectory = "";
private string _fileName = "";
private int _messagesReceived = 0;
private int _messageWrittenCount = 0;
public string TargetDirectory
{
get { return _targetDirectory; }
set { _targetDirectory = value; }
}
public string FileName
{
get { return _fileName; }
set { _fileName = value; }
}
public void pushMessage(string message)
{
_queueLock.EnterWriteLock();
_messagesReceived++;
try
{
_inputQueue.Enqueue(message);
}
catch(Exception e)
{
Logging.Logging.LogError(e.Message);
}
finally
{
_queueLock.ExitWriteLock();
}
}
public void startWorking()
{
_statusLock.EnterReadLock();
if (_working == true)
{
return;
}
_statusLock.ExitReadLock();
_statusLock.EnterWriteLock();
_working = true;
_workerThread = new Thread(new ThreadStart(work));
_workerThread.Start();
_statusLock.ExitWriteLock();
}
public void stopWorking()
{
_statusLock.EnterWriteLock();
_working = false;
_statusLock.ExitWriteLock();
if(_workerThread != null)
{
_workerThread.Join();
}
// write remaining messages if stop was called
_queueLock.EnterWriteLock();
try
{
writeQueueToFile(ref _inputQueue);
writeQueueToFile(ref _outputQueue);
Logging.Logging.LogInfo("final commit done");
}
catch(Exception e)
{
Logging.Logging.LogError(e.Message);
}
finally
{
_queueLock.ExitWriteLock();
}
Logging.Logging.LogInfo("Messages received: " + _messagesReceived);
Logging.Logging.LogInfo("Messages written: " + _messageWrittenCount);
}
private void work()
{
bool working = true;
while(working)
{
commit();
_statusLock.EnterReadLock();
working = _working;
_statusLock.ExitReadLock();
}
}
private void commit()
{
_queueLock.EnterWriteLock();
Queue<string> tmpQueue = _inputQueue;
_inputQueue = _outputQueue;
_outputQueue = tmpQueue;
_queueLock.ExitWriteLock();
writeQueueToFile(ref _outputQueue);
}
private void writeQueueToFile(ref Queue<string> messageQueue)
{
_fileLock.EnterWriteLock();
try
{
while(messageQueue.Count > 0)
{
_messageWrittenCount++;
string message = messageQueue.Dequeue();
File.AppendAllText(_targetDirectory + "\\" + _fileName, message);
}
}
catch (Exception e)
{
Logging.Logging.LogError(e.Message);
}
finally
{
_fileLock.ExitWriteLock();
}
}
}
}
ide_plugins/vs/vs2015/CoatiPlugin/CoatiPlugin/Wizard/WindowCreateCDB.cs
+34 -5
View File
@@ -1,7 +1,9 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Threading.Tasks;
using System.Threading;
using System.Windows.Forms;
namespace CoatiSoftware.CoatiPlugin.Wizard
@@ -35,6 +37,9 @@ namespace CoatiSoftware.CoatiPlugin.Wizard
private int _threadCount = 1;
private static object _lockObject = new object();
private static ReaderWriterLockSlim _readWriteLock = new ReaderWriterLockSlim();
public OnFinishedCreatingCDB CallbackOnFinishedCreatingCDB
{
get { return _onFinishedCreateCDB; }
@@ -176,6 +181,21 @@ namespace CoatiSoftware.CoatiPlugin.Wizard
private SolutionParser.CompilationDatabase CreateCommandObjects()
{
SolutionParser.CompilationDatabase cdb = new SolutionParser.CompilationDatabase();
cdb.Directory = _targetDir;
cdb.Name = _fileName;
File.WriteAllText(_targetDir + "\\" + _fileName + ".json", "");
File.AppendAllText(_targetDir + "\\" + _fileName + ".json", "[\n");
// Mutex commandObjectMutex = new Mutex();
object lockObject = new object();
object lockObject2 = new object();
Utility.QueuedFileWriter fileWriter = new Utility.QueuedFileWriter();
fileWriter.FileName = _fileName + ".json";
fileWriter.TargetDirectory = _targetDir;
fileWriter.startWorking();
try
{
@@ -185,8 +205,6 @@ namespace CoatiSoftware.CoatiPlugin.Wizard
List<Task> tasks = new List<Task>();
Object lockObject = new Object();
int projectsProcessed = 0;
foreach (EnvDTE.Project project in _projects)
{
@@ -198,34 +216,45 @@ namespace CoatiSoftware.CoatiPlugin.Wizard
{
List<SolutionParser.CommandObject> commandObjects = SolutionParser.SolutionParser.CreateCommandObjects(project, _configurationName, _platformName, _cStandard);
lock(lockObject)
lock (_lockObject)
{
projectsProcessed++;
}
float relativProgress = (float)projectsProcessed / (float)_projects.Count;
Logging.Logging.LogInfo("Processing project \"" + Logging.Obfuscation.NameObfuscator.GetObfuscatedName(project.Name) + "\"");
backgroundWorker1.ReportProgress((int)(relativProgress * 100), "Processing project \"" + project.Name + "\"");
foreach (SolutionParser.CommandObject obj in commandObjects)
{
cdb.AddOrUpdateCommandObject(obj);
// cdb.AddOrUpdateCommandObject(obj, false); // since the data is written to file right away now, no need to store it
// the cdb is however still needed to store some meta data later
fileWriter.pushMessage(obj.SerializeJSON() + ",");
}
});
tasks.Add(t);
}
int threadCount = System.Diagnostics.Process.GetCurrentProcess().Threads.Count;
Task.WaitAll(tasks.ToArray());
fileWriter.stopWorking();
backgroundWorker1.ReportProgress(100, "Writing data to file. This might take several minutes...");
}
catch(Exception e)
{
Logging.Logging.LogError("Failed to create CDB: " + e.Message);
}
finally
{
File.AppendAllText(_targetDir + "\\" + _fileName + ".json", "\n]");
}
return cdb;
}
ide_plugins/vs/vs2015/CoatiPlugin/CoatiPlugin/source.extension.vsixmanifest
+1 -1
View File
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<PackageManifest Version="2.0.0" xmlns="http://schemas.microsoft.com/developer/vsx-schema/2011" xmlns:d="http://schemas.microsoft.com/developer/vsx-schema-design/2011">
<Metadata>
<Identity Id="acf15780-03b5-440e-a41e-db79b7043fc2" Version="0.9.5" Language="en-US" Publisher="Coati Software OG" />
<Identity Id="acf15780-03b5-440e-a41e-db79b7043fc2" Version="0.9.7" Language="en-US" Publisher="Coati Software OG" />
<DisplayName>CoatiPlugin</DisplayName>
<Description xml:space="preserve">This package allows Coati to communicate with Visual Studio and vice versa.</Description>
<MoreInfo>http://www.coati.io/</MoreInfo>