diff --git a/dlt/common/storages/configuration.py b/dlt/common/storages/configuration.py index f6950873c2..f65e1a9edd 100644 --- a/dlt/common/storages/configuration.py +++ b/dlt/common/storages/configuration.py @@ -201,7 +201,7 @@ class FilesystemConfiguration(BaseConfiguration): "sftp": SFTPCredentials, } - bucket_url: str = None + bucket_url: str = "." # should be a union of all possible credentials as found in PROTOCOL_CREDENTIALS credentials: FileSystemCredentials = None diff --git a/tests/load/filesystem/test_filesystem_factory.py b/tests/load/filesystem/test_filesystem_factory.py index 5ef85ca8e9..497341bb7c 100644 --- a/tests/load/filesystem/test_filesystem_factory.py +++ b/tests/load/filesystem/test_filesystem_factory.py @@ -81,7 +81,7 @@ def test_resolve_bucket_url_ignores_foreign_credentials(): fs = filesystem() assert fs.is_hf - # without bucket_url, _resolve_bucket_url returns None and is_hf is False + # without explicit bucket_url, defaults to "." (local filesystem), so is_hf is False with custom_environ({"CREDENTIALS": "postgres://loader:password@localhost:5432/dlt_data"}): fs = filesystem() assert not fs.is_hf diff --git a/tests/sources/filesystem/test_config_sections.py b/tests/sources/filesystem/test_config_sections.py index 11caf8733d..a8a71d4080 100644 --- a/tests/sources/filesystem/test_config_sections.py +++ b/tests/sources/filesystem/test_config_sections.py @@ -1,17 +1,37 @@ import pytest +import os from dlt.sources.filesystem import filesystem, read_parquet from dlt.common.configuration.exceptions import ConfigFieldMissingException +from dlt.common.storages.configuration import FilesystemConfiguration -def test_config_sections_resolution(): +def test_config_sections_resolution_defaults_to_cwd(): + """filesystem() without bucket_url should default to current directory""" filesystem_resource = filesystem(file_glob="**/*.parquet") filesystem_pipe = filesystem_resource | read_parquet() + # Should NOT raise — defaults to "." (current working directory) + # We just verify the resource can be constructed and resolves correctly + config = FilesystemConfiguration() + assert config.bucket_url == "." + assert config.protocol == "file" + + +def test_config_sections_resolution_missing_credentials(): + """Cloud bucket_url still requires credentials — check config section path""" + filesystem_resource = filesystem( + bucket_url="s3://some-bucket", file_glob="**/*.parquet" + ) + filesystem_pipe = filesystem_resource | read_parquet() + with pytest.raises(ConfigFieldMissingException) as exc_info: list(filesystem_pipe) - # NOTE: we check that the first trace related to filesystem has the correct - # sections set - flat_trace = exc_info.value.attrs()["traces"]["bucket_url"] - assert flat_trace[0].key.startswith("SOURCES__FILESYSTEM__FILESYSTEM__") + flat_trace = exc_info.value.attrs()["traces"] + assert "credentials" in flat_trace + # Verify the correct config section path is used + assert any( + trace.sections[:3] == ["sources", "filesystem", "filesystem"] + for trace in flat_trace["credentials"] + ) \ No newline at end of file diff --git a/tests/workspace/cli/test_config_toml_writer.py b/tests/workspace/cli/test_config_toml_writer.py index 81b4c369d5..682ccd479f 100644 --- a/tests/workspace/cli/test_config_toml_writer.py +++ b/tests/workspace/cli/test_config_toml_writer.py @@ -236,11 +236,8 @@ def test_write_spec_without_defaults(example_toml) -> None: is_default_of_interest=True, ) - # bucket_url is mandatory, same for aws credentials - assert example_toml.as_string() == """[filesystem] -bucket_url = "" # fill this in! - -[filesystem.credentials] + # bucket_url now defaults to "." , same for aws credentials + assert example_toml.as_string() == """[filesystem.credentials] aws_access_key_id = "" # fill this in! aws_secret_access_key = "" # fill this in! """ @@ -296,4 +293,4 @@ class SnowflakeDatabaseConfiguration(SnowflakeClientConfiguration): ) # still here because marked specifically as of interest - assert example_toml["snowflake"]["database"] == "dlt_db" + assert example_toml["snowflake"]["database"] == "dlt_db" \ No newline at end of file