logic: refresh info generator fixes and tests

* fixed case where "to-index" state of a file has changed and thus the file needs to be re-indexed
* clear and re-index files that depend on changed non-indexed files
* clear and re-index files referenced by unchanged but cleared (due to referencing a changed file) files
* fixed: updated files with changed timestamp that do not have content in the db will now count as changed
* refined refresh info generator tests
* updated checklists for manual project setup tests

src/lib/data/storage/PersistentStorage.cpp

@@ -344,18 +344,13 @@ void PersistentStorage::clearFileElements(const std::vector<FilePath>& filePaths
 	}
 }
 
-std::vector<FileInfo> PersistentStorage::getFileInfoForAllIndexedFiles() const
+std::vector<FileInfo> PersistentStorage::getFileInfoForAllFiles() const
 {
 	TRACE();
 
 	std::vector<FileInfo> fileInfos;
 	for (StorageFile file : m_sqliteIndexStorage.getAll<StorageFile>())
 	{
-		if (!file.indexed)
-		{
-			continue;
-		}
-
 		boost::posix_time::ptime modificationTime = boost::posix_time::not_a_date_time;
 		if (file.modificationTime != "not-a-date-time")
 		{
@@ -1471,7 +1466,22 @@ std::shared_ptr<TextAccess> PersistentStorage::getFileContent(const FilePath& fi
 {
 	TRACE();
 
-	return m_sqliteIndexStorage.getFileContentByPath(filePath.wstr());
+	std::shared_ptr<TextAccess> fileContent = m_sqliteIndexStorage.getFileContentByPath(filePath.wstr());
+	if (fileContent->getLineCount() > 0)
+	{
+		return fileContent;
+	}
+	return TextAccess::createFromFile(FilePath(filePath));
+}
+
+bool PersistentStorage::hasContentForFile(const FilePath& filePath) const
+{
+	std::shared_ptr<TextAccess> fileContent = m_sqliteIndexStorage.getFileContentByPath(filePath.wstr());
+	if (fileContent->getLineCount() > 0)
+	{
+		return true;
+	}
+	return false;
 }
 
 FileInfo PersistentStorage::getFileInfoForFileId(Id id) const

src/lib/data/storage/PersistentStorage.h

@@ -66,7 +66,7 @@ public:
 	void clearAllErrors();
 	void clearFileElements(const std::vector<FilePath>& filePaths, std::function<void(int)> updateStatusCallback);
 
-	std::vector<FileInfo> getFileInfoForAllIndexedFiles() const;
+	std::vector<FileInfo> getFileInfoForAllFiles() const;
 	std::set<FilePath> getIncompleteFiles() const;
 	bool getFilePathIndexed(const FilePath& path) const;
 
@@ -119,6 +119,7 @@ public:
 	std::shared_ptr<SourceLocationFile> getCommentLocationsInFile(const FilePath& filePath) const override;
 
 	std::shared_ptr<TextAccess> getFileContent(const FilePath& filePath) const override;
+	bool hasContentForFile(const FilePath& filePath) const;
 
 	FileInfo getFileInfoForFileId(Id id) const override;
 

src/lib/data/storage/sqlite/SqliteIndexStorage.cpp

@@ -811,7 +811,7 @@ std::shared_ptr<TextAccess> SqliteIndexStorage::getFileContentByPath(const std::
 		LOG_ERROR(std::to_string(e.errorCode()) + ": " + e.errorMessage());
 	}
 
-	return TextAccess::createFromFile(FilePath(filePath));
+	return TextAccess::createFromString("");
 }
 
 void SqliteIndexStorage::setFileIndexed(Id fileId, bool indexed)

src/lib/project/RefreshInfoGenerator.cpp

@@ -12,15 +12,15 @@
 RefreshInfo RefreshInfoGenerator::getRefreshInfoForUpdatedFiles(
 	const std::vector<std::shared_ptr<SourceGroup>>& sourceGroups, std::shared_ptr<const PersistentStorage> storage)
 {
-	const std::set<FilePath> allSourceFilePathsFromSourcegroups = getAllSourceFilePaths(sourceGroups);
-
-	std::set<FilePath> unchangedFilePaths;
+	// 1) Divide filepaths that are already known by the storage to "unchanged and indexed", "unchanged and non-indexed" and "changed"
+	std::set<FilePath> unchangedIndexedFilePaths;
+	std::set<FilePath> unchangedNonindexedFilePaths;
 	std::set<FilePath> changedFilePaths;
 
 	{
-		const std::vector<FileInfo> fileInfosFromStorage = storage->getFileInfoForAllIndexedFiles();
+		const std::vector<FileInfo> fileInfosFromStorage = storage->getFileInfoForAllFiles();
 
-		std::set<FilePath> alreadyIndexedPaths;
+		std::set<FilePath> alreadyKnownPaths;
 		{
 			const std::set<FilePath> filePathsFromStorage = utility::toSet(utility::convert<FileInfo, FilePath>(
 				fileInfosFromStorage, [](const FileInfo& info) { return info.path; }
@@ -28,24 +28,35 @@ RefreshInfo RefreshInfoGenerator::getRefreshInfoForUpdatedFiles(
 
 			for (std::shared_ptr<SourceGroup> sourceGroup : sourceGroups)
 			{
-				utility::append(alreadyIndexedPaths, sourceGroup->filterToContainedFilePaths(filePathsFromStorage));
+				utility::append(alreadyKnownPaths, sourceGroup->filterToContainedFilePaths(filePathsFromStorage));
 			}
 		}
 
 		// checking source and header files
 		for (const FileInfo& info : fileInfosFromStorage)
 		{
-			if (alreadyIndexedPaths.find(info.path) != alreadyIndexedPaths.end() && info.path.exists())
+			if (alreadyKnownPaths.find(info.path) != alreadyKnownPaths.end() && info.path.exists())
 			{
-				if (didFileChange(info, storage))
+				if (storage->getFilePathIndexed(info.path))
 				{
-					changedFilePaths.insert(info.path);
+					if (didFileChange(info, storage))
+					{
+						changedFilePaths.insert(info.path);
+					}
+					else
+					{
+						unchangedIndexedFilePaths.insert(info.path);
+					}
 				}
 				else
 				{
-					unchangedFilePaths.insert(info.path);
+					changedFilePaths.insert(info.path);
 				}
 			}
+			else if (!storage->getFilePathIndexed(info.path) && !didFileChange(info, storage))
+			{
+				unchangedNonindexedFilePaths.insert(info.path);
+			}
 			else // file has been removed
 			{
 				changedFilePaths.insert(info.path);
@@ -53,56 +64,68 @@ RefreshInfo RefreshInfoGenerator::getRefreshInfoForUpdatedFiles(
 		}
 	}
 
+	const std::set<FilePath> allSourceFilePathsFromSourcegroups = getAllSourceFilePaths(sourceGroups);
+
+	// 2) Figure out which files need to be cleared
+	// 2.1) Add all changed files
 	std::set<FilePath> filesToClear = changedFilePaths;
 
-	// handle referencing paths
+	// 2.2) Add files that are reference the changed files
 	utility::append(filesToClear, storage->getReferencing(changedFilePaths));
 
-	// handle referenced paths
+	// 2.3) Handle files that are referenced by the files that will be cleared. These will be re-indexed on the fly. However, we do not
+	//		need to clear files that are also referenced by unchanged source files, because otherwise we will lose these connections.
+	// 2.3.1) Get all source file paths that will not be cleared.
+	// - Initially this list contains all source file paths the project would index right now.
+	// - Then we remove all source files that will be cleared
+	// - NOTE: Source files that are new to the project will part of this list, but won't result in any referenced
+	//   paths because they are not part of the DB. Source files that are new to the project but are already in the
+	//   DB will be removed from this list if they have changed or reference changed files.
 	std::set<FilePath> staticSourceFiles = allSourceFilePathsFromSourcegroups;
-	for (const FilePath& path : changedFilePaths)
+	for (const FilePath& path : filesToClear)
 	{
 		staticSourceFiles.erase(path);
 	}
 
+	// 2.3.2) Get sets of referenced files
 	const std::set<FilePath> staticReferencedFilePaths = storage->getReferenced(staticSourceFiles);
-	const std::set<FilePath> dynamicReferencedFilePaths = storage->getReferenced(changedFilePaths);
+	const std::set<FilePath> dynamicReferencedFilePaths = storage->getReferenced(filesToClear);
 
+	// 2.3.3) Add "dynamicReferencedFilePaths" to "filesToClear" that are not refenced by static paths, because these files may not be
+	//        referenced anymore. If they still are, they will be re-added when encountered during re-indexing.
 	for (const FilePath& path : dynamicReferencedFilePaths)
 	{
 		if (staticReferencedFilePaths.find(path) == staticReferencedFilePaths.end() &&
 			staticSourceFiles.find(path) == staticSourceFiles.end())
 		{
-			// file may not be referenced anymore and will be reindexed if still needed
 			filesToClear.insert(path);
 		}
 	}
 
-	for (const FilePath& path : unchangedFilePaths)
-	{
-		staticSourceFiles.erase(path);
-	}
-
-	const std::set<FilePath> filesToAdd = staticSourceFiles;
-
-	std::set<FilePath> staticSourceFilePaths;
+	// 3) Figure out which files need to be indexed
+	std::set<FilePath> filesToIndex;
 	for (const FilePath& path : allSourceFilePathsFromSourcegroups)
 	{
-		if (filesToClear.find(path) == filesToClear.end() && filesToAdd.find(path) == filesToAdd.end())
+		if (filesToClear.find(path) != filesToClear.end() ||							// file will be cleared
+			unchangedIndexedFilePaths.find(path) == unchangedIndexedFilePaths.end())	// file has been changed or added
 		{
-			staticSourceFilePaths.insert(path);
+			filesToIndex.insert(path);
 		}
 	}
 
+	// 4) Store and return this information
 	RefreshInfo info;
 	info.mode = REFRESH_UPDATED_FILES;
-	info.filesToClear = filesToClear;
-
-	for (const std::shared_ptr<SourceGroup>& sourceGroup : sourceGroups)
+	info.filesToIndex = filesToIndex;
+	for (const FilePath fileToClear : filesToClear)
 	{
-		if (sourceGroup->getStatus() == SOURCE_GROUP_STATUS_ENABLED)
+		if (storage->getFilePathIndexed(fileToClear))
 		{
-			utility::append(info.filesToIndex, sourceGroup->filterToContainedSourceFilePath(staticSourceFilePaths));
+			info.filesToClear.insert(fileToClear);
+		}
+		else
+		{
+			info.nonIndexedFilesToClear.insert(fileToClear);
 		}
 	}
 
@@ -115,11 +138,14 @@ RefreshInfo RefreshInfoGenerator::getRefreshInfoForIncompleteFiles(const std::ve
 	info.mode = REFRESH_UPDATED_AND_INCOMPLETE_FILES;
 
 	std::set<FilePath> incompleteFiles;
-	for (const FilePath& path : storage->getIncompleteFiles())
 	{
-		if (info.filesToClear.find(path) == info.filesToClear.end())
+		const std::set<FilePath> filesToClear = utility::concat(info.filesToClear, info.nonIndexedFilesToClear);
+		for (const FilePath& path : storage->getIncompleteFiles())
 		{
-			incompleteFiles.insert(path);
+			if (filesToClear.find(path) == filesToClear.end())
+			{
+				incompleteFiles.insert(path);
+			}
 		}
 	}
 
@@ -188,24 +214,30 @@ bool RefreshInfoGenerator::didFileChange(const FileInfo& info, std::shared_ptr<c
 	FileInfo diskFileInfo = FileSystem::getFileInfoForPath(info.path);
 	if (diskFileInfo.lastWriteTime > info.lastWriteTime)
 	{
+		if (!storage->hasContentForFile(info.path))
+		{
+			return true;
+		}
+
 		std::shared_ptr<TextAccess> storedFileContent = storage->getFileContent(info.path);
 		std::shared_ptr<TextAccess> diskFileContent = TextAccess::createFromFile(diskFileInfo.path);
 
 		const std::vector<std::string>& diskFileLines = diskFileContent->getAllLines();
 		const std::vector<std::string>& storedFileLines = storedFileContent->getAllLines();
 
-		if (diskFileLines.size() == storedFileLines.size())
+		if (diskFileLines.size() != storedFileLines.size())
 		{
-			for (size_t i = 0; i < diskFileLines.size(); i++)
-			{
-				if (diskFileLines[i] != storedFileLines[i])
-				{
-					return true;
-				}
-			}
-			return false;
+			return true;
 		}
-		return true;
+
+		for (size_t i = 0; i < diskFileLines.size(); i++)
+		{
+			if (diskFileLines[i] != storedFileLines[i])
+			{
+				return true;
+			}
+		}
+		return false;
 	}
 	return false;
 }

src/lib/project/RefreshInfoGenerator.h

@@ -15,12 +15,12 @@ class RefreshInfoGenerator
 {
 public:
 	static RefreshInfo getRefreshInfoForUpdatedFiles(
-		const std::vector<std::shared_ptr<SourceGroup>>& sourceGroups, 
+		const std::vector<std::shared_ptr<SourceGroup>>& sourceGroups,
 		std::shared_ptr<const PersistentStorage> storage
 	);
 
 	static RefreshInfo getRefreshInfoForIncompleteFiles(
-		const std::vector<std::shared_ptr<SourceGroup>>& sourceGroups, 
+		const std::vector<std::shared_ptr<SourceGroup>>& sourceGroups,
 		std::shared_ptr<const PersistentStorage> storage
 	);