Skip to content

Expose loggers logic #3132

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ognyanstoimenov wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Expose loggers logic #3132

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 99 additions & 0 deletions cpp/arcticdb/python/python_bindings_common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,11 @@

#include <pybind11/pybind11.h>
#include <arcticdb/python/python_bindings_common.hpp>
#include <arcticdb/python/python_utils.hpp>
#include <arcticdb/entity/metrics.hpp>
#include <arcticdb/log/log.hpp>
#include <arcticdb/util/preconditions.hpp>
#include <logger.pb.h>

namespace py = pybind11;

Expand Down Expand Up @@ -45,3 +48,99 @@ void register_metrics(py::module&& m, arcticdb::BindingScope scope) {
.value("PULL", arcticdb::MetricsConfig::Model::PULL)
.export_values();
}

void register_log(py::module&& log, arcticdb::BindingScope scope) {
Copy link
Copy Markdown
Collaborator Author

@ognyanstoimenov ognyanstoimenov May 22, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy-pasted from cpp/arcticdb/python/python_module.cpp. Only changed to accept BindingScope and made the exposed methods not take choose_logger by ref as it goes out of scope. Which I am not sure how it worked OK before

using arcticdb::LoggerId;
bool local_bindings = (scope == arcticdb::BindingScope::LOCAL);

log.def(
"configure",
[](const py::object& py_log_conf, bool force = false) {
arcticdb::proto::logger::LoggersConfig config;
arcticdb::python_util::pb_from_python(py_log_conf, config);
return arcticdb::log::Loggers::instance().configure(config, force);
},
py::arg("py_log_conf"),
py::arg("force") = false
);

py::enum_<spdlog::level::level_enum>(log, "LogLevel", py::module_local(local_bindings))
.value("DEBUG", spdlog::level::level_enum::debug)
.value("INFO", spdlog::level::level_enum::info)
.value("WARN", spdlog::level::level_enum::warn)
.value("ERROR", spdlog::level::level_enum::err)
.export_values();

py::enum_<LoggerId>(log, "LoggerId", py::module_local(local_bindings))
.value("ROOT", LoggerId::ROOT)
.value("STORAGE", LoggerId::STORAGE)
.value("IN_MEM", LoggerId::IN_MEM)
.value("CODEC", LoggerId::CODEC)
.value("VERSION", LoggerId::VERSION)
.value("MEMORY", LoggerId::MEMORY)
.value("TIMINGS", LoggerId::TIMINGS)
.value("LOCK", LoggerId::LOCK)
.value("SCHEDULE", LoggerId::SCHEDULE)
.value("SYMBOL", LoggerId::SYMBOL)
.value("SNAPSHOT", LoggerId::SNAPSHOT)
.export_values();

auto choose_logger = [](LoggerId log_id) -> spdlog::logger& {
switch (log_id) {
case LoggerId::STORAGE:
return arcticdb::log::storage();
case LoggerId::IN_MEM:
return arcticdb::log::inmem();
case LoggerId::CODEC:
return arcticdb::log::codec();
case LoggerId::MEMORY:
return arcticdb::log::memory();
case LoggerId::VERSION:
return arcticdb::log::version();
case LoggerId::ROOT:
return arcticdb::log::root();
case LoggerId::TIMINGS:
return arcticdb::log::timings();
case LoggerId::LOCK:
return arcticdb::log::lock();
case LoggerId::SCHEDULE:
return arcticdb::log::schedule();
case LoggerId::SYMBOL:
return arcticdb::log::symbol();
case LoggerId::SNAPSHOT:
return arcticdb::log::snapshot();
default:
arcticdb::util::raise_rte("Unsupported logger id");
}
};

log.def("log", [choose_logger](LoggerId log_id, spdlog::level::level_enum level, const std::string& msg) {
py::gil_scoped_release gil_release;
auto& logger = choose_logger(log_id);
switch (level) {
case spdlog::level::level_enum::debug:
logger.debug(msg);
break;
case spdlog::level::level_enum::info:
logger.info(msg);
break;
case spdlog::level::level_enum::warn:
logger.warn(msg);
break;
case spdlog::level::level_enum::err:
logger.error(msg);
break;
default:
arcticdb::util::raise_rte("Unsupported log level", spdlog::level::to_string_view(level));
}
});

log.def("is_active", [choose_logger](LoggerId log_id, spdlog::level::level_enum level) {
return choose_logger(log_id).should_log(level);
});

log.def("flush_all", []() {
py::gil_scoped_release gil_release;
arcticdb::log::Loggers::instance().flush_all();
});
}
3 changes: 3 additions & 0 deletions cpp/arcticdb/python/python_bindings_common.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,12 @@ class tuple;
namespace arcticdb {
enum class BindingScope : uint32_t { LOCAL = 0, GLOBAL = 1 };

enum class LoggerId { ROOT, STORAGE, IN_MEM, CODEC, VERSION, MEMORY, TIMINGS, LOCK, SCHEDULE, SYMBOL, SNAPSHOT };

pybind11::tuple to_tuple(const MetricsConfig& config);
MetricsConfig metrics_config_from_tuple(const pybind11::tuple& t);

} // namespace arcticdb

void register_metrics(pybind11::module&& m, arcticdb::BindingScope scope);
void register_log(pybind11::module&& log, arcticdb::BindingScope scope);
97 changes: 1 addition & 96 deletions cpp/arcticdb/python/python_module.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,101 +37,6 @@

namespace py = pybind11;

enum class LoggerId { ROOT, STORAGE, IN_MEM, CODEC, VERSION, MEMORY, TIMINGS, LOCK, SCHEDULE, SYMBOL, SNAPSHOT };

void register_log(py::module&& log) {
log.def(
"configure",
[](const py::object& py_log_conf, bool force = false) {
arcticdb::proto::logger::LoggersConfig config;
arcticdb::python_util::pb_from_python(py_log_conf, config);
return arcticdb::log::Loggers::instance().configure(config, force);
},
py::arg("py_log_conf"),
py::arg("force") = false
);

py::enum_<spdlog::level::level_enum>(log, "LogLevel")
.value("DEBUG", spdlog::level::level_enum::debug)
.value("INFO", spdlog::level::level_enum::info)
.value("WARN", spdlog::level::level_enum::warn)
.value("ERROR", spdlog::level::level_enum::err)
.export_values();
py::enum_<LoggerId>(log, "LoggerId")
.value("ROOT", LoggerId::ROOT)
.value("STORAGE", LoggerId::STORAGE)
.value("IN_MEM", LoggerId::IN_MEM)
.value("CODEC", LoggerId::CODEC)
.value("VERSION", LoggerId::VERSION)
.value("MEMORY", LoggerId::MEMORY)
.value("TIMINGS", LoggerId::TIMINGS)
.value("LOCK", LoggerId::LOCK)
.value("SCHEDULE", LoggerId::SCHEDULE)
.value("SYMBOL", LoggerId::SYMBOL)
.value("SNAPSHOT", LoggerId::SNAPSHOT)
.export_values();
auto choose_logger = [&](LoggerId log_id) -> decltype(arcticdb::log::storage()) /* logger ref */ {
switch (log_id) {
case LoggerId::STORAGE:
return arcticdb::log::storage();
case LoggerId::IN_MEM:
return arcticdb::log::inmem();
case LoggerId::CODEC:
return arcticdb::log::codec();
case LoggerId::MEMORY:
return arcticdb::log::memory();
case LoggerId::VERSION:
return arcticdb::log::version();
case LoggerId::ROOT:
return arcticdb::log::root();
case LoggerId::TIMINGS:
return arcticdb::log::timings();
case LoggerId::LOCK:
return arcticdb::log::lock();
case LoggerId::SCHEDULE:
return arcticdb::log::schedule();
case LoggerId::SYMBOL:
return arcticdb::log::symbol();
case LoggerId::SNAPSHOT:
return arcticdb::log::snapshot();
default:
arcticdb::util::raise_rte("Unsupported logger id");
}
};

log.def("log", [&](LoggerId log_id, spdlog::level::level_enum level, const std::string& msg) {
// assuming formatting done in python
py::gil_scoped_release gil_release;
auto& logger = choose_logger(log_id);
switch (level) {
case spdlog::level::level_enum::debug:
logger.debug(msg);
break;
case spdlog::level::level_enum::info:
logger.info(msg);
break;
case spdlog::level::level_enum::warn:
logger.warn(msg);
break;
case spdlog::level::level_enum::err:
logger.error(msg);
break;
default:
arcticdb::util::raise_rte("Unsupported log level", spdlog::level::to_string_view(level));
}
});

log.def("is_active", [&](LoggerId log_id, spdlog::level::level_enum level) {
auto& logger = choose_logger(log_id);
return logger.should_log(level);
});

log.def("flush_all", []() {
py::gil_scoped_release gil_release;
arcticdb::log::Loggers::instance().flush_all();
});
}

void register_configs_map_api(py::module& m) {
using namespace arcticdb;
#define EXPOSE_TYPE(LABEL, TYPE) \
Expand Down Expand Up @@ -351,7 +256,7 @@ PYBIND11_MODULE(arcticdb_ext, m) {
);

register_configs_map_api(m);
register_log(m.def_submodule("log"));
register_log(m.def_submodule("log"), BindingScope::GLOBAL);
register_instrumentation(m.def_submodule("instrumentation"));
register_metrics(m.def_submodule("metrics"), BindingScope::GLOBAL);
register_type_handlers();
Expand Down
89 changes: 51 additions & 38 deletions python/arcticdb/log.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,54 +8,67 @@

import traceback

from arcticdb_ext.log import configure
from arcticdb_ext.log import log as _log, LogLevel as _Lvl, LoggerId as _LoggerId, is_active as _is_active

def build_loggers(ext_log):
"""Return a logger registry bound to the given extension's log submodule.

class _Logger(object):
def __init__(self, id):
self._id = id
``ext_log`` must expose ``log``, ``LogLevel``, ``LoggerId`` and ``is_active``
(see ``register_log`` in ``python_bindings_common.cpp``).
"""
_log = ext_log.log
_Lvl = ext_log.LogLevel
_is_active = ext_log.is_active
_LoggerId = ext_log.LoggerId

def log(self, lvl, msg, *args, **kwargs):
if not self.is_active(lvl):
return
_log(self._id, lvl, msg.format(*args, **kwargs))
class _Logger:
def __init__(self, id):
self._id = id

def debug(self, msg, *args, **kwargs):
self.log(_Lvl.DEBUG, msg, *args, **kwargs)
def log(self, lvl, msg, *args, **kwargs):
if not self.is_active(lvl):
return
_log(self._id, lvl, msg.format(*args, **kwargs))

def info(self, msg, *args, **kwargs):
self.log(_Lvl.INFO, msg, *args, **kwargs)
def debug(self, msg, *args, **kwargs):
self.log(_Lvl.DEBUG, msg, *args, **kwargs)

def warning(self, msg, *args, **kwargs):
self.log(_Lvl.WARN, msg, *args, **kwargs)
def info(self, msg, *args, **kwargs):
self.log(_Lvl.INFO, msg, *args, **kwargs)

def is_active(self, lvl):
return _is_active(self._id, lvl)
def warning(self, msg, *args, **kwargs):
self.log(_Lvl.WARN, msg, *args, **kwargs)

warn = warning
def is_active(self, lvl):
return _is_active(self._id, lvl)

def error(self, msg, *args, **kwargs):
self.log(_Lvl.ERROR, msg, *args, **kwargs)
warn = warning

def exception(self, msg, *args, **kwargs):
exc = traceback.format_exc()
_log(self._id, _Lvl.ERROR, msg.format(*args, **kwargs) + "\n" + exc)
def error(self, msg, *args, **kwargs):
self.log(_Lvl.ERROR, msg, *args, **kwargs)

def exception(self, msg, *args, **kwargs):
exc = traceback.format_exc()
_log(self._id, _Lvl.ERROR, msg.format(*args, **kwargs) + "\n" + exc)

logger_by_name = {
"codec": _Logger(_LoggerId.CODEC),
"inmem": _Logger(_LoggerId.IN_MEM),
"root": _Logger(_LoggerId.ROOT),
"storage": _Logger(_LoggerId.STORAGE),
"version": _Logger(_LoggerId.VERSION),
"memory": _Logger(_LoggerId.MEMORY),
"timings": _Logger(_LoggerId.TIMINGS),
"lock": _Logger(_LoggerId.LOCK),
"schedule": _Logger(_LoggerId.SCHEDULE),
"symbol": _Logger(_LoggerId.SYMBOL),
"snapshot": _Logger(_LoggerId.SNAPSHOT),
}
return {
"codec": _Logger(_LoggerId.CODEC),
"inmem": _Logger(_LoggerId.IN_MEM),
"root": _Logger(_LoggerId.ROOT),
"storage": _Logger(_LoggerId.STORAGE),
"version": _Logger(_LoggerId.VERSION),
"memory": _Logger(_LoggerId.MEMORY),
"timings": _Logger(_LoggerId.TIMINGS),
"lock": _Logger(_LoggerId.LOCK),
"schedule": _Logger(_LoggerId.SCHEDULE),
"symbol": _Logger(_LoggerId.SYMBOL),
"snapshot": _Logger(_LoggerId.SNAPSHOT),
}

for key, value in logger_by_name.items():
globals()[key] = value

import arcticdb_ext.log as _ext_log

configure = _ext_log.configure
logger_by_name = build_loggers(_ext_log)

for _key, _value in logger_by_name.items():
globals()[_key] = _value
Loading