ynkmr/Sourcetrail
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

data: indexer processes

Browse Source 
* TaskBuildIndex starts seperate indexer processes and communicates via shared memory
* indexer processes get restarted if they fail
* indexer processes are killed when the app closes or crashes
* added utility class SharedMemory to allocate and access shared memory, providing basic data structures
* added SharedMemoryGarbageCollector for keeping track of running instances and cleaning up shared memory in case of a crash
* show errors for source files that crashed during indexing
* added basic exception handling to task scheduling
* added option to turn off processes and use threads in preferences, but still use shared memory

fortune cookie message = Good news from someone dear is coming soon.
This commit is contained in:
Eberhard Graether
2017-05-04 15:50:59 +02:00
parent 9beea94fd8
commit 706119ebcc
109 changed files with 3351 additions and 1261 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/data/IntermediateStorage.h
+38 -12
View File
@@ -16,6 +16,8 @@ public:
virtual ~IntermediateStorage();
void clear();
size_t getByteSize() const;
size_t getSourceLocationCount() const;
void setAllFilesIncomplete();
@@ -32,17 +34,45 @@ public:
virtual void addCommentLocation(Id fileNodeId, uint startLine, uint startCol, uint endLine, uint endCol);
virtual void addError(const std::string& message, const FilePath& filePath, uint startLine, uint startCol, bool fatal, bool indexed);
virtual void forEachNode(std::function<void(const Id /*id*/, const StorageNode& /*data*/)> callback) const;
virtual void forEachNode(std::function<void(const StorageNode& /*data*/)> callback) const;
virtual void forEachFile(std::function<void(const StorageFile& /*data*/)> callback) const;
virtual void forEachSymbol(std::function<void(const StorageSymbol& /*data*/)> callback) const;
virtual void forEachEdge(std::function<void(const Id /*id*/, const StorageEdge& /*data*/)> callback) const;
virtual void forEachLocalSymbol(std::function<void(const Id /*id*/, const StorageLocalSymbol& /*data*/)> callback) const;
virtual void forEachSourceLocation(std::function<void(const Id /*id*/, const StorageSourceLocation& /*data*/)> callback) const;
virtual void forEachEdge(std::function<void(const StorageEdge& /*data*/)> callback) const;
virtual void forEachLocalSymbol(std::function<void(const StorageLocalSymbol& /*data*/)> callback) const;
virtual void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const;
virtual void forEachOccurrence(std::function<void(const StorageOccurrence& /*data*/)> callback) const;
virtual void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const;
virtual void forEachCommentLocation(std::function<void(const StorageCommentLocation& /*data*/)> callback) const;
virtual void forEachError(std::function<void(const StorageError& /*data*/)> callback) const;
// for conversion to and from 'SharedIntermediateStorage'
std::vector<StorageNode> getStorageNodes() const;
std::vector<StorageFile> getStorageFiles() const;
std::vector<StorageSymbol> getStorageSymbols() const;
std::vector<StorageEdge> getStorageEdges() const;
std::vector<StorageLocalSymbol> getStorageLocalSymbols() const;
std::vector<StorageSourceLocation> getStorageSourceLocations() const;
std::vector<StorageOccurrence> getStorageOccurrences() const;
std::vector<StorageComponentAccess> getComponentAccesses() const;
std::vector<StorageCommentLocation> getCommentLocations() const;
std::vector<StorageError> getErrors() const;
void setStorageNodes(const std::vector<StorageNode>& storageNodes);
void setStorageFiles(const std::vector<StorageFile>& storageFiles);
void setStorageSymbols(const std::vector<StorageSymbol>& storageSymbols);
void setStorageEdges(const std::vector<StorageEdge>& storageEdges);
void setStorageLocalSymbols(const std::vector<StorageLocalSymbol>& storageLocalSymbols);
void setStorageSourceLocations(const std::vector<StorageSourceLocation>& storageSourceLocations);
void setStorageOccurrences(const std::vector<StorageOccurrence>& storageOccurrences);
void setComponentAccesses(const std::vector<StorageComponentAccess>& componentAccesses);
void setCommentLocations(const std::vector<StorageCommentLocation>& commentLocations);
void setErrors(const std::vector<StorageError>& errors);
Id getNextId() const;
void setNextId(const Id nextId);
private:
std::string serialize(const StorageNode& node) const;
std::string serialize(const StorageFile& file) const;
@@ -54,22 +84,18 @@ private:
std::string serialize(const StorageCommentLocation& commentLocation) const;
std::string serialize(const StorageError& error) const;
std::unordered_map<std::string, Id> m_nodeNamesToIds; // this is used to prevent duplicates (unique)
std::map<Id, std::shared_ptr<StorageNode>> m_nodeIdsToData;
std::unordered_map<std::string, StorageNode> m_nodes;
std::unordered_set<std::string> m_serializedFiles; // this is used to prevent duplicates (unique)
std::vector<StorageFile> m_files;
std::vector<StorageSymbol> m_symbols;
std::unordered_map<std::string, Id> m_edgeNamesToIds; // this is used to prevent duplicates (unique)
std::map<Id, std::shared_ptr<StorageEdge>> m_edgeIdsToData;
std::unordered_map<std::string, StorageEdge> m_edges;
std::unordered_map<std::string, Id> m_localSymbolNamesToIds; // this is used to prevent duplicates (unique)
std::map<Id, std::shared_ptr<StorageLocalSymbol>> m_localSymbolIdsToData;
std::unordered_map<std::string, StorageLocalSymbol> m_localSymbols;
std::unordered_map<std::string, Id> m_sourceLocationNamesToIds; // this is used to prevent duplicates (unique)
std::map<Id, std::shared_ptr<StorageSourceLocation>> m_sourceLocationIdsToData;
std::unordered_map<std::string, StorageSourceLocation> m_sourceLocations;
std::unordered_set<std::string> m_serializedOccurrences; // this is used to prevent duplicates (unique)
std::vector<StorageOccurrence> m_occurrences;