diff --git a/llm/cli.py b/llm/cli.py index ecd8121a7..38f07eeaa 100644 --- a/llm/cli.py +++ b/llm/cli.py @@ -3258,11 +3258,14 @@ def get_db(): if not content: if not input or input == "-": # Read from stdin - input_source = sys.stdin.buffer if binary else sys.stdin - content = input_source.read() + if binary: + content = sys.stdin.buffer.read() + else: + content = sys.stdin.buffer.read().decode("utf-8") else: mode = "rb" if binary else "r" - with open(input, mode) as f: + kwargs = {} if binary else {"encoding": "utf-8"} + with open(input, mode, **kwargs) as f: content = f.read() if not content: @@ -3570,11 +3573,14 @@ def similar(collection, id, input, content, binary, number, plain, database, pre if not content: if not input or input == "-": # Read from stdin - input_source = sys.stdin.buffer if binary else sys.stdin - content = input_source.read() + if binary: + content = sys.stdin.buffer.read() + else: + content = sys.stdin.buffer.read().decode("utf-8") else: mode = "rb" if binary else "r" - with open(input, mode) as f: + kwargs = {} if binary else {"encoding": "utf-8"} + with open(input, mode, **kwargs) as f: content = f.read() if not content: raise click.ClickException("No content provided") diff --git a/tests/test_embed_cli.py b/tests/test_embed_cli.py index 11b67f56c..295ca1c54 100644 --- a/tests/test_embed_cli.py +++ b/tests/test_embed_cli.py @@ -347,6 +347,24 @@ def test_embed_multi_file_input(tmpdir, use_stdin, prefix, prepend, filename, co assert ids == expected_ids +def test_embed_file_utf8_content_preserved(tmpdir, user_path): + """Non-ASCII UTF-8 characters in a file should be stored without corruption.""" + db_path = str(user_path / "embeddings.db") + path = tmpdir / "content.txt" + # Write raw UTF-8 bytes to guarantee the on-disk encoding + path.write_binary("æ ø å".encode("utf-8")) + runner = CliRunner() + result = runner.invoke( + cli, + ["embed", "items", "1", "-i", str(path), "-m", "embed-demo", "--store", "-d", db_path], + catch_exceptions=False, + ) + assert result.exit_code == 0 + db = sqlite_utils.Database(db_path) + row = list(db["embeddings"].rows)[0] + assert row["content"] == "æ ø å" + + def test_embed_multi_files_binary_store(tmpdir): db_path = tmpdir / "embeddings.db" args = ["embed-multi", "binfiles", "-d", str(db_path), "-m", "embed-demo"]