data: parser performance optimization for large projects

* moved inserting everything into the database to the end of parsing a file. this reduces many duplicate queries. * removed ParserClient interface from Storage. * errors are updated after parsing a file because we only if there are any duplicate errors after merging the new errors into the database. * renamed field in db * added fatal column to error db
2016-03-10 18:24:28 +01:00
parent e7c7178057
commit 9a7b818dba
17 changed files with 1254 additions and 1055 deletions
@@ -0,0 +1,63 @@
+#ifndef INTERMEDIATE_STORAGE_H
+#define INTERMEDIATE_STORAGE_H
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+
+#include "utility/types.h"
+#include "data/name/NameHierarchy.h"
+#include "data/parser/ParseLocation.h"
+
+#include "data/SqliteStorage.h"
+#include "data/StorageTypes.h"
+
+class IntermediateStorage
+{
+public:
+	IntermediateStorage();
+	~IntermediateStorage();
+	Id addEdge(int type, Id sourceId, Id targetId);
+	Id addNode(int type, const NameHierarchy& nameHierarchy, bool defined);
+	Id addFile(const std::string& name, const std::string& filePath, const std::string& modificationTime);
+	Id addFile(const std::string& filePath);
+	void addSourceLocation(Id elementId, const ParseLocation& location, bool isScope);
+	void addComponentAccess(Id nodeId , int type);
+	void addCommentLocation(const ParseLocation& location);
+	void addError(const std::string& message, bool fatal, const ParseLocation& location);
+
+	void transferToStorage(SqliteStorage& storage); // TODO: remove this and use foreach-callbacks instead
+
+	void forEachFile(std::function<void(const Id /*id*/, const StorageFile& /*data*/)> callback) const;
+	void forEachNode(std::function<void(const Id /*id*/, const StorageNode& /*data*/)> callback) const;
+	void forEachEdge(std::function<void(const Id /*id*/, const StorageEdge& /*data*/)> callback) const;
+	void forEachSourceLocation(std::function<void(const StorageSourceLocation& /*data*/)> callback) const;
+	void forEachComponentAccess(std::function<void(const StorageComponentAccess& /*data*/)> callback) const;
+	void forEachCommentLocation(std::function<void(const StorageCommentLocation& /*data*/)> callback) const;
+	void forEachError(std::function<void(const StorageError& /*data*/)> callback) const;
+
+private:
+	std::string serialize(const StorageEdge& edge);
+	std::string serialize(const StorageNode& node);
+	std::string serialize(const StorageFile& file);
+
+	std::unordered_map<std::string, Id> m_fileNamesToIds; // this is used to prevent duplicates (unique)
+	std::unordered_map<Id, std::shared_ptr<StorageFile>> m_fileIdsToData;
+
+	std::unordered_map<std::string, Id> m_nodeNamesToIds; // this is used to prevent duplicates (unique)
+	std::unordered_map<Id, std::shared_ptr<StorageNode>> m_nodeIdsToData;
+
+	std::unordered_map<std::string, Id> m_edgeNamesToIds; // this is used to prevent duplicates (unique)
+	std::unordered_map<Id, std::shared_ptr<StorageEdge>> m_edgeIdsToData;
+
+	std::vector<StorageSourceLocation> m_sourceLocations;
+	std::vector<StorageComponentAccess> m_componentAccesses;
+	std::vector<StorageCommentLocation> m_commentLocations;
+	std::vector<StorageError> m_errors;
+
+	std::unordered_map<Id, Id> m_nodeIdsToMemberEdgeIds;
+
+	Id m_nextId;
+};
+
+#endif // INTERMEDIATE_STORAGE_H