Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dlt/common/storages/configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@ class FilesystemConfiguration(BaseConfiguration):
"sftp": SFTPCredentials,
}

bucket_url: str = None
bucket_url: str = "."

# should be a union of all possible credentials as found in PROTOCOL_CREDENTIALS
credentials: FileSystemCredentials = None
Expand Down
2 changes: 1 addition & 1 deletion tests/load/filesystem/test_filesystem_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def test_resolve_bucket_url_ignores_foreign_credentials():
fs = filesystem()
assert fs.is_hf

# without bucket_url, _resolve_bucket_url returns None and is_hf is False
# without explicit bucket_url, defaults to "." (local filesystem), so is_hf is False
with custom_environ({"CREDENTIALS": "postgres://loader:password@localhost:5432/dlt_data"}):
fs = filesystem()
assert not fs.is_hf
Expand Down
30 changes: 25 additions & 5 deletions tests/sources/filesystem/test_config_sections.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,37 @@
import pytest
import os

from dlt.sources.filesystem import filesystem, read_parquet
from dlt.common.configuration.exceptions import ConfigFieldMissingException
from dlt.common.storages.configuration import FilesystemConfiguration


def test_config_sections_resolution():
def test_config_sections_resolution_defaults_to_cwd():
"""filesystem() without bucket_url should default to current directory"""
filesystem_resource = filesystem(file_glob="**/*.parquet")
filesystem_pipe = filesystem_resource | read_parquet()

# Should NOT raise — defaults to "." (current working directory)
# We just verify the resource can be constructed and resolves correctly
config = FilesystemConfiguration()
assert config.bucket_url == "."
assert config.protocol == "file"


def test_config_sections_resolution_missing_credentials():
"""Cloud bucket_url still requires credentials — check config section path"""
filesystem_resource = filesystem(
bucket_url="s3://some-bucket", file_glob="**/*.parquet"
)
filesystem_pipe = filesystem_resource | read_parquet()

with pytest.raises(ConfigFieldMissingException) as exc_info:
list(filesystem_pipe)

# NOTE: we check that the first trace related to filesystem has the correct
# sections set
flat_trace = exc_info.value.attrs()["traces"]["bucket_url"]
assert flat_trace[0].key.startswith("SOURCES__FILESYSTEM__FILESYSTEM__")
flat_trace = exc_info.value.attrs()["traces"]
assert "credentials" in flat_trace
# Verify the correct config section path is used
assert any(
trace.sections[:3] == ["sources", "filesystem", "filesystem"]
for trace in flat_trace["credentials"]
)
9 changes: 3 additions & 6 deletions tests/workspace/cli/test_config_toml_writer.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,11 +236,8 @@ def test_write_spec_without_defaults(example_toml) -> None:
is_default_of_interest=True,
)

# bucket_url is mandatory, same for aws credentials
assert example_toml.as_string() == """[filesystem]
bucket_url = "<configure me>" # fill this in!

[filesystem.credentials]
# bucket_url now defaults to "." , same for aws credentials
assert example_toml.as_string() == """[filesystem.credentials]
aws_access_key_id = "<configure me>" # fill this in!
aws_secret_access_key = "<configure me>" # fill this in!
"""
Expand Down Expand Up @@ -296,4 +293,4 @@ class SnowflakeDatabaseConfiguration(SnowflakeClientConfiguration):
)

# still here because marked specifically as of interest
assert example_toml["snowflake"]["database"] == "dlt_db"
assert example_toml["snowflake"]["database"] == "dlt_db"
Loading