Nodejs/node
1
0
Fork 0
mirror of https://github.com/nodejs/node.git synced 2024-11-21 10:59:27 +00:00
Code Issues Actions 25 Packages Projects Releases Wiki Activity

src: update compile cache storage structure

Browse Source 
This refactors the compile cache handler in preparation for the
JS API, and updates the compile cache storage structure into:

- $NODE_COMPILE_CACHE_DIR
  - $NODE_VERION-$ARCH-$CACHE_DATA_VERSION_TAG-$UID
    - $FILENAME_AND_MODULE_TYPE_HASH.cache

This also adds a magic number to the beginning of the cache
files for verification, and returns the status, compile
cache directory and/or error message of enabling the
compile cache in a structure, which can be converted as
JS counterparts by the upcoming JS API.

PR-URL: https://github.com/nodejs/node/pull/54291
Refs: https://github.com/nodejs/node/issues/53639
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Ethan Arrowood <ethan@arrowood.dev>
Reviewed-By: Chengzhong Wu <legendecas@gmail.com>
This commit is contained in:
Joyee Cheung 2024-08-19 13:54:36 +02:00 committed by GitHub
parent 4f94397650
commit b246f22554
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 137 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
src/compile_cache.cc
View File

@ -1,4 +1,5 @@
#include "compile_cache.h"
#include <string>
#include "debug_utils-inl.h"
#include "env-inl.h"
#include "node_file.h"
@ -27,15 +28,19 @@ uint32_t GetHash(const char* data, size_t size) {
return crc32(crc, reinterpret_cast<const Bytef*>(data), size);
}
uint32_t GetCacheVersionTag() {
std::string_view node_version(NODE_VERSION);
uint32_t v8_tag = v8::ScriptCompiler::CachedDataVersionTag();
uLong crc = crc32(0L, Z_NULL, 0);
crc = crc32(crc, reinterpret_cast<const Bytef*>(&v8_tag), sizeof(uint32_t));
crc = crc32(crc,
reinterpret_cast<const Bytef*>(node_version.data()),
node_version.size());
return crc;
std::string GetCacheVersionTag() {
// On platforms where uids are available, use different folders for
// different users to avoid cache miss due to permission incompatibility.
// On platforms where uids are not available, bare with the cache miss.
// This should be fine on Windows, as there local directories tend to be
// user-specific.
std::string tag = std::string(NODE_VERSION) + '-' + std::string(NODE_ARCH) +
'-' +
Uint32ToHex(v8::ScriptCompiler::CachedDataVersionTag());
#ifdef NODE_IMPLEMENTS_POSIX_CREDENTIALS
tag += '-' + std::to_string(getuid());
#endif
return tag;
}
uint32_t GetCacheKey(std::string_view filename, CachedCodeType type) {
@ -63,6 +68,10 @@ v8::ScriptCompiler::CachedData* CompileCacheEntry::CopyCache() const {
data, cache_size, v8::ScriptCompiler::CachedData::BufferOwned);
}
// Used for identifying and verifying a file is a compile cache file.
// See comments in CompileCacheHandler::Persist().
constexpr uint32_t kCacheMagicNumber = 0x8adfdbb2;
void CompileCacheHandler::ReadCacheFile(CompileCacheEntry* entry) {
Debug("[compile cache] reading cache from %s for %s %s...",
entry->cache_filename,
@ -100,12 +109,20 @@ void CompileCacheHandler::ReadCacheFile(CompileCacheEntry* entry) {
return;
}
Debug("[%d %d %d %d]...",
Debug("[%d %d %d %d %d]...",
headers[kMagicNumberOffset],
headers[kCodeSizeOffset],
headers[kCacheSizeOffset],
headers[kCodeHashOffset],
headers[kCacheHashOffset]);
if (headers[kMagicNumberOffset] != kCacheMagicNumber) {
Debug("magic number mismatch: expected %d, actual %d\n",
kCacheMagicNumber,
headers[kMagicNumberOffset]);
return;
}
// Check the code size and hash which are already computed.
if (headers[kCodeSizeOffset] != entry->code_size) {
Debug("code size mismatch: expected %d, actual %d\n",
@ -202,11 +219,14 @@ CompileCacheEntry* CompileCacheHandler::GetOrInsert(
compiler_cache_store_.emplace(key, std::make_unique<CompileCacheEntry>());
auto* result = emplaced.first->second.get();
std::u8string cache_filename_u8 =
(compile_cache_dir_ / Uint32ToHex(key)).u8string();
result->code_hash = code_hash;
result->code_size = code_utf8.length();
result->cache_key = key;
result->cache_filename =
(compile_cache_dir_ / Uint32ToHex(result->cache_key)).string();
std::string(cache_filename_u8.begin(), cache_filename_u8.end()) +
".cache";
result->source_filename = filename_utf8.ToString();
result->cache = nullptr;
result->type = type;
@ -264,6 +284,7 @@ void CompileCacheHandler::MaybeSave(CompileCacheEntry* entry,
}
// Layout of a cache file:
// [uint32_t] magic number
// [uint32_t] code size
// [uint32_t] code hash
// [uint32_t] cache size
@ -301,14 +322,16 @@ void CompileCacheHandler::Persist() {
// Generating headers.
std::vector<uint32_t> headers(kHeaderCount);
headers[kMagicNumberOffset] = kCacheMagicNumber;
headers[kCodeSizeOffset] = entry->code_size;
headers[kCacheSizeOffset] = cache_size;
headers[kCodeHashOffset] = entry->code_hash;
headers[kCacheHashOffset] = cache_hash;
Debug("[compile cache] writing cache for %s in %s [%d %d %d %d]...",
Debug("[compile cache] writing cache for %s in %s [%d %d %d %d %d]...",
entry->source_filename,
entry->cache_filename,
headers[kMagicNumberOffset],
headers[kCodeSizeOffset],
headers[kCacheSizeOffset],
headers[kCodeHashOffset],
@ -335,53 +358,63 @@ CompileCacheHandler::CompileCacheHandler(Environment* env)
// Directory structure:
// - Compile cache directory (from NODE_COMPILE_CACHE)
// - <cache_version_tag_1>: hash of CachedDataVersionTag + NODE_VERESION
// - <cache_version_tag_2>
// - <cache_version_tag_3>
// - <cache_file_1>: a hash of filename + module type
// - <cache_file_2>
// - <cache_file_3>
bool CompileCacheHandler::InitializeDirectory(Environment* env,
const std::string& dir) {
compiler_cache_key_ = GetCacheVersionTag();
std::string compiler_cache_key_string = Uint32ToHex(compiler_cache_key_);
std::vector<std::string_view> paths = {dir, compiler_cache_key_string};
std::string cache_dir = PathResolve(env, paths);
// - $NODE_VERION-$ARCH-$CACHE_DATA_VERSION_TAG-$UID
// - $FILENAME_AND_MODULE_TYPE_HASH.cache: a hash of filename + module type
CompileCacheEnableResult CompileCacheHandler::Enable(Environment* env,
const std::string& dir) {
std::string cache_tag = GetCacheVersionTag();
std::string absolute_cache_dir_base = PathResolve(env, {dir});
std::filesystem::path cache_dir_with_tag =
std::filesystem::path(absolute_cache_dir_base) / cache_tag;
std::u8string cache_dir_with_tag_u8 = cache_dir_with_tag.u8string();
std::string cache_dir_with_tag_str(cache_dir_with_tag_u8.begin(),
cache_dir_with_tag_u8.end());
CompileCacheEnableResult result;
Debug("[compile cache] resolved path %s + %s -> %s\n",
dir,
compiler_cache_key_string,
cache_dir);
cache_tag,
cache_dir_with_tag_str);
if (UNLIKELY(!env->permission()->is_granted(
env, permission::PermissionScope::kFileSystemWrite, cache_dir))) {
Debug("[compile cache] skipping cache because write permission for %s "
"is not granted\n",
cache_dir);
return false;
env,
permission::PermissionScope::kFileSystemWrite,
cache_dir_with_tag_str))) {
result.message = "Skipping compile cache because write permission for " +
cache_dir_with_tag_str + " is not granted";
result.status = CompileCacheEnableStatus::kFailed;
return result;
}
if (UNLIKELY(!env->permission()->is_granted(
env, permission::PermissionScope::kFileSystemRead, cache_dir))) {
Debug("[compile cache] skipping cache because read permission for %s "
"is not granted\n",
cache_dir);
return false;
env,
permission::PermissionScope::kFileSystemRead,
cache_dir_with_tag_str))) {
result.message = "Skipping compile cache because read permission for " +
cache_dir_with_tag_str + " is not granted";
result.status = CompileCacheEnableStatus::kFailed;
return result;
}
fs::FSReqWrapSync req_wrap;
int err = fs::MKDirpSync(nullptr, &(req_wrap.req), cache_dir, 0777, nullptr);
int err = fs::MKDirpSync(
nullptr, &(req_wrap.req), cache_dir_with_tag_str, 0777, nullptr);
if (is_debug_) {
Debug("[compile cache] creating cache directory %s...%s\n",
cache_dir,
cache_dir_with_tag_str,
err < 0 ? uv_strerror(err) : "success");
}
if (err != 0 && err != UV_EEXIST) {
return false;
result.message =
"Cannot create cache directory: " + std::string(uv_strerror(err));
result.status = CompileCacheEnableStatus::kFailed;
return result;
}
compile_cache_dir_ = std::filesystem::path(cache_dir);
return true;
compile_cache_dir_str_ = absolute_cache_dir_base;
result.cache_directory = absolute_cache_dir_base;
compile_cache_dir_ = cache_dir_with_tag;
result.status = CompileCacheEnableStatus::kEnabled;
return result;
}
} // namespace node

36
src/compile_cache.h
View File

@ -7,6 +7,7 @@
#include <filesystem>
#include <memory>
#include <string>
#include <string_view>
#include <unordered_map>
#include "v8.h"
@ -34,10 +35,27 @@ struct CompileCacheEntry {
v8::ScriptCompiler::CachedData* CopyCache() const;
};
#define COMPILE_CACHE_STATUS(V) \
V(kFailed) /* Failed to enable the cache */ \
V(kEnabled) /* Was not enabled before, and now enabled. */ \
V(kAlreadyEnabled) /* Was already enabled. */
enum class CompileCacheEnableStatus : uint8_t {
#define V(status) status,
COMPILE_CACHE_STATUS(V)
#undef V
};
struct CompileCacheEnableResult {
CompileCacheEnableStatus status;
std::string cache_directory;
std::string message; // Set in case of failure.
};
class CompileCacheHandler {
public:
explicit CompileCacheHandler(Environment* env);
bool InitializeDirectory(Environment* env, const std::string& dir);
CompileCacheEnableResult Enable(Environment* env, const std::string& dir);
void Persist();
@ -50,6 +68,7 @@ class CompileCacheHandler {
void MaybeSave(CompileCacheEntry* entry,
v8::Local<v8::Module> mod,
bool rejected);
std::string_view cache_dir() { return compile_cache_dir_str_; }
private:
void ReadCacheFile(CompileCacheEntry* entry);
@ -62,19 +81,18 @@ class CompileCacheHandler {
template <typename... Args>
inline void Debug(const char* format, Args&&... args) const;
static constexpr size_t kCodeSizeOffset = 0;
static constexpr size_t kCacheSizeOffset = 1;
static constexpr size_t kCodeHashOffset = 2;
static constexpr size_t kCacheHashOffset = 3;
static constexpr size_t kHeaderCount = 4;
static constexpr size_t kMagicNumberOffset = 0;
static constexpr size_t kCodeSizeOffset = 1;
static constexpr size_t kCacheSizeOffset = 2;
static constexpr size_t kCodeHashOffset = 3;
static constexpr size_t kCacheHashOffset = 4;
static constexpr size_t kHeaderCount = 5;
v8::Isolate* isolate_ = nullptr;
bool is_debug_ = false;
std::string compile_cache_dir_str_;
std::filesystem::path compile_cache_dir_;
// The compile cache is stored in a directory whose name is the hex string of
// compiler_cache_key_.
uint32_t compiler_cache_key_ = 0;
std::unordered_map<uint32_t, std::unique_ptr<CompileCacheEntry>>
compiler_cache_store_;
};

37
src/env.cc
View File

@ -1118,15 +1118,36 @@ void Environment::InitializeCompileCache() {
dir_from_env.empty()) {
return;
}
auto handler = std::make_unique<CompileCacheHandler>(this);
if (handler->InitializeDirectory(this, dir_from_env)) {
compile_cache_handler_ = std::move(handler);
AtExit(
[](void* env) {
static_cast<Environment*>(env)->compile_cache_handler()->Persist();
},
this);
EnableCompileCache(dir_from_env);
}
CompileCacheEnableResult Environment::EnableCompileCache(
const std::string& cache_dir) {
CompileCacheEnableResult result;
if (!compile_cache_handler_) {
std::unique_ptr<CompileCacheHandler> handler =
std::make_unique<CompileCacheHandler>(this);
result = handler->Enable(this, cache_dir);
if (result.status == CompileCacheEnableStatus::kEnabled) {
compile_cache_handler_ = std::move(handler);
AtExit(
[](void* env) {
static_cast<Environment*>(env)->compile_cache_handler()->Persist();
},
this);
}
if (!result.message.empty()) {
Debug(this,
DebugCategory::COMPILE_CACHE,
"[compile cache] %s\n",
result.message);
}
} else {
result.status = CompileCacheEnableStatus::kAlreadyEnabled;
result.cache_directory = compile_cache_handler_->cache_dir();
}
return result;
}
void Environment::ExitEnv(StopFlags::Flags flags) {

3
src/env.h
View File

@ -1024,6 +1024,9 @@ class Environment final : public MemoryRetainer {
inline CompileCacheHandler* compile_cache_handler();
inline bool use_compile_cache() const;
void InitializeCompileCache();
// Enable built-in compile cache if it has not yet been enabled.
// The cache will be persisted to disk on exit.
CompileCacheEnableResult EnableCompileCache(const std::string& cache_dir);
void RunAndClearNativeImmediates(bool only_refed = false);
void RunAndClearInterrupts();

6
test/parallel/test-compile-cache-permission-disallowed.js
View File

@ -39,7 +39,7 @@ function testDisallowed(dummyDir, cacheDirInPermission, cacheDirInEnv) {
},
{
stderr(output) {
assert.match(output, /skipping cache because write permission for .* is not granted/);
assert.match(output, /Skipping compile cache because write permission for .* is not granted/);
return true;
}
});
@ -63,7 +63,7 @@ function testDisallowed(dummyDir, cacheDirInPermission, cacheDirInEnv) {
},
{
stderr(output) {
assert.match(output, /skipping cache because write permission for .* is not granted/);
assert.match(output, /Skipping compile cache because write permission for .* is not granted/);
return true;
}
});
@ -86,7 +86,7 @@ function testDisallowed(dummyDir, cacheDirInPermission, cacheDirInEnv) {
},
{
stderr(output) {
assert.match(output, /skipping cache because read permission for .* is not granted/);
assert.match(output, /Skipping compile cache because read permission for .* is not granted/);
return true;
}
});