Skip to content
This repository was archived by the owner on Jul 30, 2020. It is now read-only.

Commit db4dae6

Browse files
decimaddecimad
committed
- try to limit unqlite memory buffer to 32*1024 pages (=128MiB)
- formatting
1 parent 204f9d2 commit db4dae6

10 files changed

+239
-247
lines changed

‎src/cache_manager.cc‎

Lines changed: 164 additions & 168 deletions
Original file line numberDiff line numberDiff line change
@@ -7,231 +7,227 @@
77

88
#include <loguru/loguru.hpp>
99

10-
1110
#include <algorithm>
1211
#include <unordered_map>
1312

14-
#include "unqlite.h"
1513
#include "query.h"
14+
#include "unqlite.h"
1615

1716
namespace {
1817

19-
//Storing index+content in a file directory (possibly shared between multiple cquery caches, since it could be a user-setting)
20-
//The key already has to contain the distinction between content and index
21-
struct FileBasedCacheDriver : public ICacheStore
22-
{
23-
FileBasedCacheDriver(Config* config, std::string projectDir, std::string externalsDir)
24-
: projectDir_(projectDir), externalsDir_(externalsDir)
25-
{
18+
// Storing index+content in a file directory (possibly shared between multiple
19+
// cquery caches, since it could be a user-setting) The key already has to
20+
// contain the distinction between content and index
21+
struct FileBasedCacheDriver : public ICacheStore {
22+
FileBasedCacheDriver(Config* config,
23+
std::string projectDir,
24+
std::string externalsDir)
25+
: projectDir_(projectDir), externalsDir_(externalsDir) {}
26+
27+
std::string KeyToFilePath(const std::string& key) {
28+
assert(!g_config->cacheDirectory.empty());
29+
std::string cache_file;
30+
31+
size_t len = g_config->projectRoot.size();
32+
if (StartsWith(key, g_config->projectRoot)) {
33+
cache_file = projectDir_ + '/' + EscapeFileName(key.substr(len));
34+
} else {
35+
cache_file = externalsDir_ + '/' + EscapeFileName(key);
2636
}
2737

28-
std::string KeyToFilePath(const std::string& key)
29-
{
30-
assert(!g_config->cacheDirectory.empty());
31-
std::string cache_file;
38+
return g_config->cacheDirectory + cache_file;
39+
}
3240

33-
size_t len = g_config->projectRoot.size();
34-
if (StartsWith(key, g_config->projectRoot)) {
35-
cache_file = projectDir_ + '/' + EscapeFileName(key.substr(len));
36-
} else {
37-
cache_file = externalsDir_ + '/' + EscapeFileName(key);
38-
}
41+
optional<std::string> Read(const std::string& key) override {
42+
std::string file_path = KeyToFilePath(key);
43+
return ReadContent(file_path);
44+
}
3945

40-
return g_config->cacheDirectory + cache_file;
41-
}
46+
void Write(const std::string& key, const std::string& value) override {
47+
std::string file_path = KeyToFilePath(key);
48+
WriteToFile(file_path, value);
49+
}
4250

43-
optional<std::string> Read(const std::string& key) override
44-
{
45-
std::string file_path = KeyToFilePath(key);
46-
return ReadContent(file_path);
47-
}
48-
49-
void Write(const std::string& key, const std::string& value) override
50-
{
51-
std::string file_path = KeyToFilePath(key);
52-
WriteToFile(file_path, value);
53-
}
54-
55-
private:
56-
std::string projectDir_;
57-
std::string externalsDir_;
51+
private:
52+
std::string projectDir_;
53+
std::string externalsDir_;
5854
};
5955

60-
void UnqliteHandleResult(std::string operation, unqlite* database, int ret)
61-
{
62-
if (ret != UNQLITE_OK)
63-
{
64-
const char *zBuf;
65-
int iLen;
66-
unqlite_config(database,UNQLITE_CONFIG_ERR_LOG,&zBuf,&iLen);
67-
LOG_S(WARNING) << "Unqlite error: \"" << std::string(zBuf, zBuf+iLen) << "\". Rolling back.";
68-
unqlite_rollback(database);
69-
}
56+
void UnqliteHandleResult(std::string operation, unqlite* database, int ret) {
57+
if (ret != UNQLITE_OK) {
58+
const char* zBuf;
59+
int iLen;
60+
unqlite_config(database, UNQLITE_CONFIG_ERR_LOG, &zBuf, &iLen);
61+
LOG_S(WARNING) << "Unqlite error: \"" << std::string(zBuf, zBuf + iLen)
62+
<< "\". Rolling back.";
63+
unqlite_rollback(database);
64+
}
7065
}
7166

67+
// Storing index+content in an unqlite database (possibly shared between
68+
// multiple cquery caches, since it could be a user-setting)
69+
struct UnqliteCacheDriver : public ICacheStore {
70+
UnqliteCacheDriver(unqlite* database) : database_(database) {}
71+
72+
UnqliteCacheDriver(UnqliteCacheDriver&) = delete;
7273

73-
// Storing index+content in an unqlite database (possibly shared between multiple cquery caches, since it could be a user-setting)
74-
struct UnqliteCacheDriver : public ICacheStore
75-
{
76-
UnqliteCacheDriver(unqlite* database)
77-
: database_(database)
78-
{}
79-
80-
UnqliteCacheDriver(UnqliteCacheDriver&) = delete;
81-
82-
optional<std::string> Read(const std::string& key) override
83-
{
84-
unqlite_int64 valueLength;
85-
std::string result;
86-
int ret = unqlite_kv_fetch(database_, key.data(), key.size(), nullptr, &valueLength);
87-
88-
if (ret == UNQLITE_OK)
89-
{
90-
result.resize(valueLength);
91-
ret = unqlite_kv_fetch(database_, key.data(), key.size(), const_cast<char*>(result.data()), &valueLength);
92-
}
93-
94-
if (ret == UNQLITE_OK) return std::move(result);
95-
else return {};
74+
optional<std::string> Read(const std::string& key) override {
75+
unqlite_int64 valueLength;
76+
std::string result;
77+
int ret = unqlite_kv_fetch(database_, key.data(), key.size(), nullptr,
78+
&valueLength);
79+
80+
if (ret == UNQLITE_OK) {
81+
result.resize(valueLength);
82+
ret = unqlite_kv_fetch(database_, key.data(), key.size(),
83+
const_cast<char*>(result.data()), &valueLength);
9684
}
9785

98-
void Write(const std::string& key, const std::string& value) override
99-
{
100-
int ret;
101-
while((ret = unqlite_kv_store(database_, key.data(), key.size(), value.data(), value.size())) == UNQLITE_BUSY);
102-
if (ret != UNQLITE_OK)
103-
{
104-
UnqliteHandleResult("unqlite_kv_store", database_, ret);
105-
}
86+
if (ret == UNQLITE_OK)
87+
return std::move(result);
88+
else
89+
return {};
90+
}
91+
92+
void Write(const std::string& key, const std::string& value) override {
93+
int ret;
94+
while ((ret = unqlite_kv_store(database_, key.data(), key.size(),
95+
value.data(), value.size())) == UNQLITE_BUSY)
96+
;
97+
if (ret != UNQLITE_OK) {
98+
UnqliteHandleResult("unqlite_kv_store", database_, ret);
10699
}
100+
}
107101

108-
void Close() override
109-
{
110-
LOG_S(INFO) << "Unqlite: Closing the store.";
111-
int ret;
112-
while((ret = unqlite_close(database_)) == UNQLITE_BUSY);
102+
void Close() override {
103+
LOG_S(INFO) << "Unqlite: Closing the store.";
104+
int ret;
105+
while ((ret = unqlite_close(database_)) == UNQLITE_BUSY)
106+
;
113107

114-
if (ret != UNQLITE_OK)
115-
{
116-
UnqliteHandleResult("unqlite_close", database_, ret);
117-
}
108+
if (ret != UNQLITE_OK) {
109+
UnqliteHandleResult("unqlite_close", database_, ret);
118110
}
111+
}
119112

120-
~UnqliteCacheDriver() override
121-
{
122-
}
123-
124-
unqlite* database_;
113+
~UnqliteCacheDriver() override {}
114+
115+
unqlite* database_;
125116
};
126117

127118
std::string SerializationFormatToSuffix(SerializeFormat format) {
128-
switch (format) {
129-
case SerializeFormat::Json:
130-
return ".json";
131-
case SerializeFormat::MessagePack:
132-
return ".mpack";
133-
}
134-
assert(false);
135-
return ".json";
119+
switch (format) {
120+
case SerializeFormat::Json:
121+
return ".json";
122+
case SerializeFormat::MessagePack:
123+
return ".mpack";
124+
}
125+
assert(false);
126+
return ".json";
136127
}
137128

138-
}
129+
} // namespace
139130

140131
IndexCache::IndexCache(std::shared_ptr<ICacheStore> driver)
141-
: driver_(std::move(driver))
142-
{
143-
}
144-
145-
// Tries to recover an index file (content+serialized index) for a given source file from the cache store and returns a
146-
// non-owning reference or null, buffering the IndexFile internally for later take
147-
IndexFile* IndexCache::TryLoad(const NormalizedPath& path)
148-
{
149-
IndexFile* result = nullptr;
150-
auto ptr = LoadIndexFileFromCache(path);
151-
if (ptr) {
152-
result = ptr.get();
153-
caches_.emplace(path.path, std::move(ptr));
154-
}
155-
156-
return result;
132+
: driver_(std::move(driver)) {}
133+
134+
// Tries to recover an index file (content+serialized index) for a given source
135+
// file from the cache store and returns a non-owning reference or null,
136+
// buffering the IndexFile internally for later take
137+
IndexFile* IndexCache::TryLoad(const NormalizedPath& path) {
138+
IndexFile* result = nullptr;
139+
auto ptr = LoadIndexFileFromCache(path);
140+
if (ptr) {
141+
result = ptr.get();
142+
caches_.emplace(path.path, std::move(ptr));
143+
}
144+
145+
return result;
157146
}
158147

159-
// Tries to recover an index file (content+serialized index) for a given source file from the cache store and returns a
160-
// non-owning reference or null
161-
std::unique_ptr<IndexFile> IndexCache::TryTakeOrLoad(const NormalizedPath& path)
162-
{
163-
return LoadIndexFileFromCache(path);
148+
// Tries to recover an index file (content+serialized index) for a given source
149+
// file from the cache store and returns a non-owning reference or null
150+
std::unique_ptr<IndexFile> IndexCache::TryTakeOrLoad(
151+
const NormalizedPath& path) {
152+
return LoadIndexFileFromCache(path);
164153
}
165154

166155
// Only load the buffered file content from the cache
167-
optional<std::string> IndexCache::TryLoadContent(const NormalizedPath& path)
168-
{
169-
return driver_->Read(path.path);
156+
optional<std::string> IndexCache::TryLoadContent(const NormalizedPath& path) {
157+
return driver_->Read(path.path);
170158
}
171159

172-
std::unique_ptr<IndexFile> IndexCache::LoadIndexFileFromCache(const NormalizedPath& file)
173-
{
174-
optional<std::string> file_content = ReadContent(file.path);
175-
optional<std::string> serialized_indexed_content = ReadContent(file.path + SerializationFormatToSuffix(g_config->cacheFormat));
176-
177-
if (!file_content || !serialized_indexed_content)
178-
return nullptr;
160+
std::unique_ptr<IndexFile> IndexCache::LoadIndexFileFromCache(
161+
const NormalizedPath& file) {
162+
optional<std::string> file_content = ReadContent(file.path);
163+
optional<std::string> serialized_indexed_content = ReadContent(
164+
file.path + SerializationFormatToSuffix(g_config->cacheFormat));
179165

180-
return Deserialize(g_config->cacheFormat, file.path, *serialized_indexed_content,
181-
*file_content, IndexFile::kMajorVersion);
182-
}
166+
if (!file_content || !serialized_indexed_content)
167+
return nullptr;
183168

169+
return Deserialize(g_config->cacheFormat, file.path,
170+
*serialized_indexed_content, *file_content,
171+
IndexFile::kMajorVersion);
172+
}
184173

185174
// Write an IndexFile to the cache storage
186-
void IndexCache::Write(IndexFile& file)
187-
{
188-
driver_->Write(file.path, file.file_contents);
189-
driver_->Write(file.path + SerializationFormatToSuffix(g_config->cacheFormat), Serialize(g_config->cacheFormat, file));
175+
void IndexCache::Write(IndexFile& file) {
176+
driver_->Write(file.path, file.file_contents);
177+
driver_->Write(file.path + SerializationFormatToSuffix(g_config->cacheFormat),
178+
Serialize(g_config->cacheFormat, file));
190179
}
191180

192181
// Iterate over all loaded caches.
193-
void IndexCache::IterateLoadedCaches(std::function<void(IndexFile*)> fn)
194-
{
195-
for( auto& file : caches_ )
196-
{
197-
fn(file.second.get());
198-
}
182+
void IndexCache::IterateLoadedCaches(std::function<void(IndexFile*)> fn) {
183+
for (auto& file : caches_) {
184+
fn(file.second.get());
185+
}
199186
}
200187

201-
std::shared_ptr<IndexCache> MakeIndexCache(std::shared_ptr<ICacheStore> store)
202-
{
203-
return std::make_shared<IndexCache>(std::move(store));
188+
std::shared_ptr<IndexCache> MakeIndexCache(std::shared_ptr<ICacheStore> store) {
189+
return std::make_shared<IndexCache>(std::move(store));
204190
}
205191

206192
// Returns null if the given root path does not exist
207-
std::shared_ptr<ICacheStore> OpenOrConnectFileStore(const NormalizedPath& path)
208-
{
209-
const std::string projectDirName = EscapeFileName(g_config->projectRoot);
210-
const std::string externalsDirName = '@' + EscapeFileName(g_config->projectRoot);
193+
std::shared_ptr<ICacheStore> OpenOrConnectFileStore(
194+
const NormalizedPath& path) {
195+
const std::string projectDirName = EscapeFileName(g_config->projectRoot);
196+
const std::string externalsDirName =
197+
'@' + EscapeFileName(g_config->projectRoot);
211198

212-
MakeDirectoryRecursive(g_config->cacheDirectory + projectDirName);
213-
MakeDirectoryRecursive(g_config->cacheDirectory + externalsDirName);
199+
MakeDirectoryRecursive(g_config->cacheDirectory + projectDirName);
200+
MakeDirectoryRecursive(g_config->cacheDirectory + externalsDirName);
214201

215-
LOG_S(INFO) << "Connecting to file storage in directory \"" << path.path << "\".";
202+
LOG_S(INFO) << "Connecting to file storage in directory \"" << path.path
203+
<< "\".";
216204

217-
return std::make_shared<FileBasedCacheDriver>(g_config, projectDirName, externalsDirName);
205+
return std::make_shared<FileBasedCacheDriver>(g_config, projectDirName,
206+
externalsDirName);
218207
}
219208

220209
// Return null if the given file path does not exists and cannot be created
221-
std::shared_ptr<ICacheStore> OpenOrConnectUnqliteStore(const NormalizedPath& path_to_db)
222-
{
223-
std::string databaseFile = g_config->cacheDirectory + EscapeFileName(g_config->projectRoot) + ".db";
224-
unqlite* database = nullptr;
210+
std::shared_ptr<ICacheStore> OpenOrConnectUnqliteStore(
211+
const NormalizedPath& path_to_db) {
212+
std::string databaseFile =
213+
g_config->cacheDirectory + EscapeFileName(g_config->projectRoot) + ".db";
214+
unqlite* database = nullptr;
225215

226-
LOG_S(INFO) << "Connecting to unqlite storage in directory \"" << databaseFile << "\".";
216+
LOG_S(INFO) << "Connecting to unqlite storage in directory \"" << databaseFile
217+
<< "\".";
227218

228-
int ret = unqlite_open(&database, databaseFile.c_str(), UNQLITE_OPEN_CREATE);
229-
230-
if (ret != UNQLITE_OK)
231-
LOG_S(WARNING) << "Unqlite: unqlite_open reported error condition " << ret << ".";
219+
int ret = unqlite_open(&database, databaseFile.c_str(), UNQLITE_OPEN_CREATE);
232220

233-
//if (ret == UNQLITE_OK) return std::make_shared<UnqliteCacheDriver>(database);
234-
if (ret == UNQLITE_OK) return std::shared_ptr<ICacheStore>(new UnqliteCacheDriver(database) );
235-
else return nullptr;
236-
}
221+
if (ret != UNQLITE_OK)
222+
LOG_S(WARNING) << "Unqlite: unqlite_open reported error condition " << ret
223+
<< ".";
224+
225+
ret = unqlite_config(database, UNQLITE_CONFIG_MAX_PAGE_CACHE, 64*1024);
237226

227+
// if (ret == UNQLITE_OK) return
228+
// std::make_shared<UnqliteCacheDriver>(database);
229+
if (ret == UNQLITE_OK)
230+
return std::shared_ptr<ICacheStore>(new UnqliteCacheDriver(database));
231+
else
232+
return nullptr;
233+
}

0 commit comments

Comments
 (0)