ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

logic: reduce access to filesystem while indexing

Browse Source 
* use canonical filepath cache in cxx indexer
* made constructors of FilePath explicit
* use FilePath at more places instead of string
* forward declares FilePath wherever possible
This commit is contained in:
malte_langkabel
2017-05-04 10:55:16 +02:00
parent 55d3f9c189
commit 054b8fab17
156 changed files with 656 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/IntermediateStorage.cpp
+3 -3
View File
@@ -48,15 +48,15 @@ void IntermediateStorage::setAllFilesIncomplete()
void IntermediateStorage::setFilesWithErrorsIncomplete()
{
std::set<FilePath> errorFiles;
std::set<std::string> errorFileNames;
for (StorageError& error : m_errors)
{
errorFiles.insert(error.filePath);
errorFileNames.insert(error.filePath.str());
}
for (StorageFile& file : m_files)
{
if (errorFiles.find(file.filePath) != errorFiles.end())
if (errorFileNames.find(file.filePath) != errorFileNames.end())
{
file.complete = false;
}
src/lib/data/PersistentStorage.cpp
+5 -3
View File
@@ -4,6 +4,8 @@
#include <queue>
#include "utility/Cache.h"
#include "utility/file/FileInfo.h"
#include "utility/file/FilePath.h"
#include "utility/file/FileSystem.h"
#include "utility/logging/logging.h"
#include "utility/messaging/type/MessageNewErrors.h"
@@ -1273,7 +1275,7 @@ std::shared_ptr<SourceLocationCollection> PersistentStorage::getSourceLocationsF
for (const StorageFile& file : m_sqliteIndexStorage.getAllByIds<StorageFile>(fileIds))
{
collection->addSourceLocationFile(m_sqliteIndexStorage.getSourceLocationsForFile(file.filePath));
collection->addSourceLocationFile(m_sqliteIndexStorage.getSourceLocationsForFile(FilePath(file.filePath)));
}
if (nonFileIds.size())
@@ -1351,7 +1353,7 @@ std::shared_ptr<SourceLocationFile> PersistentStorage::getSourceLocationsForFile
}
std::shared_ptr<SourceLocationFile> PersistentStorage::getSourceLocationsForLinesInFile(
const std::string& filePath, uint firstLineNumber, uint lastLineNumber
const FilePath& filePath, uint firstLineNumber, uint lastLineNumber
) const
{
TRACE();
@@ -2041,7 +2043,7 @@ void PersistentStorage::buildSearchIndex()
auto it = fileMap.find(node.id);
if (it != fileMap.end())
{
FilePath filePath = it->second.filePath;
FilePath filePath(it->second.filePath);
if (filePath.exists())
{
src/lib/data/PersistentStorage.h
+1 -3
View File
@@ -4,8 +4,6 @@
#include <memory>
#include <vector>
#include "utility/file/FilePath.h"
#include "data/access/StorageAccess.h"
#include "data/fulltextsearch/FullTextSearchIndex.h"
#include "data/graph/token_component/TokenComponentAccess.h"
@@ -125,7 +123,7 @@ public:
virtual std::shared_ptr<SourceLocationFile> getSourceLocationsForFile(const FilePath& filePath) const;
virtual std::shared_ptr<SourceLocationFile> getSourceLocationsForLinesInFile(
const std::string& filePath, uint firstLineNumber, uint lastLineNumber
const FilePath& filePath, uint firstLineNumber, uint lastLineNumber
) const;
virtual std::shared_ptr<SourceLocationFile> getCommentLocationsInFile(const FilePath& filePath) const;
src/lib/data/SqliteBookmarkStorage.h
-1
View File
@@ -8,7 +8,6 @@
#include "data/bookmark/NodeBookmark.h"
#include "data/SqliteStorage.h"
#include "data/StorageTypes.h"
#include "utility/file/FilePath.h"
#include "utility/types.h"
class SqliteBookmarkStorage
src/lib/data/SqliteIndexStorage.cpp
+3 -3
View File
@@ -74,7 +74,7 @@ void SqliteIndexStorage::addFile(const int id, const std::string& filePath, cons
return;
}
std::shared_ptr<TextAccess> content = TextAccess::createFromFile(filePath);
std::shared_ptr<TextAccess> content = TextAccess::createFromFile(FilePath(filePath));
const size_t lineCount = content->getLineCount();
const bool success = executeStatement(
@@ -589,7 +589,7 @@ std::shared_ptr<TextAccess> SqliteIndexStorage::getFileContentByPath(const std::
LOG_ERROR(std::to_string(e.errorCode()) + ": " + e.errorMessage());
}
return TextAccess::createFromFile(filePath);
return TextAccess::createFromFile(FilePath(filePath));
}
void SqliteIndexStorage::setFileComplete(bool complete, Id fileId)
@@ -1176,7 +1176,7 @@ std::vector<StorageError> SqliteIndexStorage::doGetAll<StorageError>(const std::
if (lineNumber != -1 && columnNumber != -1)
{
errors.push_back(StorageError(id, message, filePath, lineNumber, columnNumber, fatal, indexed));
errors.push_back(StorageError(id, message, FilePath(filePath), lineNumber, columnNumber, fatal, indexed));
id++;
}
src/lib/data/SqliteIndexStorage.h
+2 -3
View File
@@ -7,14 +7,13 @@
#include "data/location/SourceLocationFile.h"
#include "data/name/NameHierarchy.h"
#include "data/StorageTypes.h"
#include "data/SqliteDatabaseIndex.h"
#include "utility/file/FilePath.h"
#include "data/SqliteStorage.h"
#include "data/StorageTypes.h"
#include "utility/types.h"
#include "utility/utility.h"
#include "utility/utilityString.h"
#include "data/SqliteStorage.h"
class TextAccess;
class Version;
src/lib/data/TaskCleanStorage.cpp
+1
View File
@@ -2,6 +2,7 @@
#include "component/view/DialogView.h"
#include "data/PersistentStorage.h"
#include "utility/file/FilePath.h"
#include "utility/scheduling/Blackboard.h"
#include "utility/utility.h"
#include "Application.h"
src/lib/data/TaskCleanStorage.h
+1 -1
View File
@@ -3,11 +3,11 @@
#include <vector>
#include "utility/file/FilePath.h"
#include "utility/scheduling/Task.h"
#include "utility/TimePoint.h"
class DialogView;
class FilePath;
class PersistentStorage;
class TaskCleanStorage
src/lib/data/TaskFinishParsing.h
-1
View File
@@ -3,7 +3,6 @@
#include <vector>
#include "utility/file/FilePath.h"
#include "utility/scheduling/Task.h"
class DialogView;
src/lib/data/access/StorageAccess.h
+3 -3
View File
@@ -6,8 +6,6 @@
#include <vector>
#include "utility/types.h"
#include "utility/file/FileInfo.h"
#include "utility/file/FilePath.h"
#include "data/bookmark/Bookmark.h"
#include "data/bookmark/BookmarkCategory.h"
@@ -20,6 +18,8 @@
#include "data/ErrorInfo.h"
#include "data/StorageStats.h"
class FilePath;
struct FileInfo;
class Graph;
class SourceLocationCollection;
class SourceLocationFile;
@@ -61,7 +61,7 @@ public:
const std::vector<Id>& locationIds) const = 0;
virtual std::shared_ptr<SourceLocationFile> getSourceLocationsForFile(const FilePath& filePath) const = 0;
virtual std::shared_ptr<SourceLocationFile> getSourceLocationsForLinesInFile(
const std::string& filePath, uint firstLineNumber, uint lastLineNumber) const = 0;
const FilePath& filePath, uint firstLineNumber, uint lastLineNumber) const = 0;
virtual std::shared_ptr<SourceLocationFile> getCommentLocationsInFile(const FilePath& filePath) const = 0;
src/lib/data/access/StorageAccessProxy.cpp
+6 -5
View File
@@ -4,8 +4,9 @@
#include "data/location/SourceLocationCollection.h"
#include "data/location/SourceLocationFile.h"
#include "utility/logging/logging.h"
#include "utility/file/FileInfo.h"
#include "utility/file/FilePath.h"
#include "utility/logging/logging.h"
#include "utility/messaging/type/MessageShowErrors.h"
#include "utility/TimePoint.h"
@@ -224,11 +225,11 @@ std::shared_ptr<SourceLocationFile> StorageAccessProxy::getSourceLocationsForFil
return m_subject->getSourceLocationsForFile(filePath);
}
return std::make_shared<SourceLocationFile>("", false, false);
return std::make_shared<SourceLocationFile>(FilePath(), false, false);
}
std::shared_ptr<SourceLocationFile> StorageAccessProxy::getSourceLocationsForLinesInFile(
const std::string& filePath, uint firstLineNumber, uint lastLineNumber
const FilePath& filePath, uint firstLineNumber, uint lastLineNumber
) const
{
if (hasSubject())
@@ -236,7 +237,7 @@ std::shared_ptr<SourceLocationFile> StorageAccessProxy::getSourceLocationsForLin
return m_subject->getSourceLocationsForLinesInFile(filePath, firstLineNumber, lastLineNumber);
}
return std::make_shared<SourceLocationFile>("", false, false);
return std::make_shared<SourceLocationFile>(FilePath(), false, false);
}
std::shared_ptr<SourceLocationFile> StorageAccessProxy::getCommentLocationsInFile(const FilePath& filePath) const
@@ -246,7 +247,7 @@ std::shared_ptr<SourceLocationFile> StorageAccessProxy::getCommentLocationsInFil
return m_subject->getCommentLocationsInFile(filePath);
}
return std::make_shared<SourceLocationFile>("", false, false);
return std::make_shared<SourceLocationFile>(FilePath(), false, false);
}
std::shared_ptr<TextAccess> StorageAccessProxy::getFileContent(const FilePath& filePath) const
src/lib/data/access/StorageAccessProxy.h
+1 -1
View File
@@ -51,7 +51,7 @@ public:
) const;
virtual std::shared_ptr<SourceLocationFile> getSourceLocationsForFile(const FilePath& filePath) const;
virtual std::shared_ptr<SourceLocationFile> getSourceLocationsForLinesInFile(
const std::string& filePath, uint firstLineNumber, uint lastLineNumber
const FilePath& filePath, uint firstLineNumber, uint lastLineNumber
) const;
virtual std::shared_ptr<SourceLocationFile> getCommentLocationsInFile(const FilePath& filePath) const;
src/lib/data/location/SourceLocationFile.h
+1
View File
@@ -2,6 +2,7 @@
#define SOURCE_LOCATION_FILE_H
#include <map>
#include <memory>
#include <ostream>
#include <set>
src/lib/data/parser/Parser.h
-2
View File
@@ -5,8 +5,6 @@
#include <string>
#include <vector>
#include "utility/file/FilePath.h"
class ParserClient;
class TextAccess;