Skip to content

Add a mechanism to silence noisy s3 simulator logs #2945

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
G-D-Petrov wants to merge 7 commits into
base: master
Choose a base branch
from
Open

Add a mechanism to silence noisy s3 simulator logs #2945

Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
782576c
Add a way to silence noisy s3 simulator logs
G-D-Petrov Mar 2, 2026
b73aac1
Restore mark file
G-D-Petrov Mar 2, 2026
d338177
Apply suggestions from code review
G-D-Petrov Mar 2, 2026
8697186
Add contextlib import to s3.py for improved resource management
G-D-Petrov Mar 2, 2026
9028893
Fix errors
G-D-Petrov Mar 2, 2026
c46a499
Refactor S3 server logging configuration to use strtobool for environ…
G-D-Petrov Mar 2, 2026
18b9b3a
Apply suggestions from code review
G-D-Petrov Mar 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions python/arcticdb/storage_fixtures/s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
from typing import Optional, Any, Type

import werkzeug
from werkzeug.serving import WSGIRequestHandler
import botocore.exceptions
from moto.server import DomainDispatcherApplication, create_backend_app

Expand Down Expand Up @@ -56,6 +57,40 @@
logger = logging.getLogger("S3 Storage Fixture")


def _is_truthy_env(var_name: str) -> bool:
Comment thread
G-D-Petrov marked this conversation as resolved.
Outdated
return os.getenv(var_name, "").strip().lower() in {"1", "true", "yes", "on"}


def _configure_moto_server_logging(verbose: bool) -> None:
if verbose:
return

werkzeug_logger = logging.getLogger("werkzeug")
werkzeug_logger.propagate = False
werkzeug_logger.setLevel(logging.ERROR)


def _suppress_moto_server_stdio(verbose: bool) -> None:
if verbose:
return

with open(os.devnull, "w") as devnull:
os.dup2(devnull.fileno(), sys.stdout.fileno())
os.dup2(devnull.fileno(), sys.stderr.fileno())


class _QuietMotoRequestHandler(WSGIRequestHandler):
def log(self, type, message, *args):
return

def log_request(self, code="-", size="-"):
return

def log_error(self, format, *args):
Comment thread
G-D-Petrov marked this conversation as resolved.
super().log_error(format, *args)
return


class Key:
def __init__(self, *, id: str, secret: str, user_name: str):
self.id = id
Expand Down Expand Up @@ -769,22 +804,32 @@ def __call__(self, environ, start_response):


def run_s3_server(port, key_file, cert_file):
verbose = _is_truthy_env("ARCTICDB_MOTO_VERBOSE")
_configure_moto_server_logging(verbose)
Comment thread
G-D-Petrov marked this conversation as resolved.
_suppress_moto_server_stdio(verbose)
request_handler = None if verbose else _QuietMotoRequestHandler
werkzeug.run_simple(
"0.0.0.0",
port,
HostDispatcherApplication(create_backend_app),
threaded=True,
ssl_context=(cert_file, key_file) if cert_file and key_file else None,
request_handler=request_handler,
)


def run_gcp_server(port, key_file, cert_file):
verbose = _is_truthy_env("ARCTICDB_MOTO_VERBOSE")
_configure_moto_server_logging(verbose)
_suppress_moto_server_stdio(verbose)
request_handler = None if verbose else _QuietMotoRequestHandler
werkzeug.run_simple(
"0.0.0.0",
port,
GcpHostDispatcherApplication(create_backend_app),
threaded=True,
ssl_context=(cert_file, key_file) if cert_file and key_file else None,
request_handler=request_handler,
)


Expand Down
Loading