-
Notifications
You must be signed in to change notification settings - Fork 96
[API Compatibility] Change compatibility apis to ChangePrefixMatcher, inject paddle.enable_compat(), fix test environment context pollution -part #895
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from 12 commits
321f32b
9bf7bfb
204c7a1
edb4042
1ff4284
020ce44
e584efa
bfff52a
3dcd2a1
81a3c07
8170d4f
f04bbf3
9bef7f9
838a8a5
f7e1c04
5fe7de1
14d8aeb
e0808c4
6bd002d
f8f344a
961af0c
d8f6e9d
62739d7
7bb47d1
67be297
1f774a4
9c52c23
e39b042
8ee2f99
22d59d6
7f11a54
5362b97
1f99f24
d9a9a74
5cd351e
770552a
38925e2
9091b39
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -46,6 +46,9 @@ def __init__( | |
| self.imports_map[self.file]["api_alias_name_map"] = {} | ||
| self.insert_pass_node = set() | ||
| self.change_prefix_api_map = defaultdict(set) | ||
| # Set in visit_Module: True when this file actually imports torch (so we | ||
| # added paddle imports) and therefore needs paddle.enable_compat injected. | ||
| self.need_enable_compat = False | ||
|
|
||
| def visit_Import(self, node): | ||
| """ | ||
|
|
@@ -525,3 +528,94 @@ def visit_Module(self, node): | |
| (self.root, "body", 0), ast.parse(f"import {paddle_package}").body | ||
| ) | ||
| line_NO += 1 | ||
|
|
||
| # enable_compat is injected in transform() (needs all imports in the body). | ||
| # Gate on real torch imports, not paddle_package_list: the latter also holds | ||
| # MAY_TORCH packages (os/einops/setuptools) that need no compat switch. | ||
| if self.imports_map[self.file]["torch_packages"]: | ||
| self.need_enable_compat = True | ||
|
|
||
| def transform(self): | ||
| super(ImportTransformer, self).transform() | ||
| self._inject_enable_compat() | ||
|
|
||
| @staticmethod | ||
| def _is_future_import(node): | ||
| return isinstance(node, ast.ImportFrom) and node.module == "__future__" | ||
|
|
||
| @staticmethod | ||
| def _is_import(node): | ||
| return isinstance(node, (ast.Import, ast.ImportFrom)) | ||
|
|
||
| @staticmethod | ||
| def _is_docstring(node): | ||
| return ( | ||
| isinstance(node, ast.Expr) | ||
| and isinstance(node.value, ast.Constant) | ||
| and isinstance(node.value.value, str) | ||
| ) | ||
|
|
||
| @staticmethod | ||
| def _is_enable_compat_call(node): | ||
| return ( | ||
| isinstance(node, ast.Expr) | ||
| and isinstance(node.value, ast.Call) | ||
| and isinstance(node.value.func, ast.Attribute) | ||
| and node.value.func.attr == "enable_compat" | ||
| and isinstance(node.value.func.value, ast.Name) | ||
| and node.value.func.value.id == "paddle" | ||
| ) | ||
|
|
||
| @staticmethod | ||
| def _binds_paddle(node): | ||
| # `import paddle` or `import paddle.xxx` (no asname) binds the name `paddle` | ||
| if isinstance(node, ast.Import): | ||
| for alias_node in node.names: | ||
| if alias_node.asname is None and ( | ||
| alias_node.name == "paddle" or alias_node.name.startswith("paddle.") | ||
| ): | ||
| return True | ||
| return False | ||
|
|
||
| def _inject_enable_compat(self): | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这个使用record_scope不能插入吗?插入这个不需要从0开始重写吧 |
||
| """Insert ``paddle.enable_compat(level=2)`` after the docstring, | ||
| ``__future__`` imports and the import block (level=2 aliases the | ||
| torch-aligned ``paddle.compat.*`` APIs onto ``paddle.*``). Post-pass so it | ||
| is never placed above a ``__future__`` import (which is a SyntaxError). | ||
| """ | ||
| if not self.need_enable_compat: | ||
| return | ||
|
|
||
| body = [n for n in self.root.body if not self._is_enable_compat_call(n)] | ||
|
|
||
| # hoist all __future__ imports (they must precede every other statement) | ||
| futures = [n for n in body if self._is_future_import(n)] | ||
| body = [n for n in body if not self._is_future_import(n)] | ||
|
|
||
| # hoist the module docstring if only imports precede it (we prepend imports) | ||
| doc = [] | ||
| for i, node in enumerate(body): | ||
| if self._is_docstring(node) and all(self._is_import(n) for n in body[:i]): | ||
| doc = [node] | ||
| body = body[:i] + body[i + 1 :] | ||
| break | ||
|
|
||
| # split off the contiguous import block at the top of the remainder | ||
| end = 0 | ||
| while end < len(body) and self._is_import(body[end]): | ||
| end += 1 | ||
| imports, rest = body[:end], body[end:] | ||
|
|
||
| # enable_compat needs the name `paddle` bound (submodule-only aliases may | ||
| # not bind it, e.g. `import torch.nn as nn` -> `import paddle.nn as nn`) | ||
| if not any(self._binds_paddle(n) for n in imports): | ||
| imports = ast.parse("import paddle").body + imports | ||
|
|
||
| compat = ast.parse("paddle.enable_compat(level=2)").body | ||
| self.root.body = doc + futures + imports + compat + rest | ||
| ast.fix_missing_locations(self.root) | ||
| log_info( | ||
| self.logger, | ||
| "add 'paddle.enable_compat(level=2)' after imports", | ||
| self.file_name, | ||
| ) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -20,6 +20,9 @@ | |
| import numpy as np | ||
|
|
||
| sys.path.append(os.path.dirname(__file__) + "/..") | ||
| sys.path.append(os.path.dirname(__file__)) | ||
|
|
||
| from conftest import disable_paddle_compat | ||
|
|
||
| from paconvert.converter import Converter | ||
|
|
||
|
|
@@ -85,6 +88,7 @@ def run( | |
| ) | ||
| assert paddle_code == expect_paddle_code, error_msg | ||
| elif compared_tensor_names: | ||
| disable_paddle_compat() | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这里为何需要disable
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 你这里是不是应该从eval执行的地方去修改,适配好torch、paddle交替执行的情况
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 你这里应该从eval执行的地方去修改,适配torch、paddle交替执行的情况 |
||
| pytorch_ns = {} | ||
| try: | ||
| exec(pytorch_code, pytorch_ns) | ||
|
|
@@ -117,6 +121,7 @@ def run( | |
| except Exception as e: | ||
| raise AssertionError(f"Unable to align results: {e}") | ||
| else: | ||
| disable_paddle_compat() | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这里为何需要disable
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 同上 |
||
| pytorch_ns = {} | ||
| try: | ||
| exec(pytorch_code, pytorch_ns) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| # Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved. | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 这个文件还是不能删除吗
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 我目前还没有办法删去 |
||
| # | ||
| # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| # you may not use this file except in compliance with the License. | ||
| # You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, software | ||
| # distributed under the License is distributed on an "AS IS" BASIS, | ||
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| # See the License for the specific language governing permissions and | ||
| # limitations under the License. | ||
|
|
||
| import sys | ||
|
|
||
| import pytest | ||
|
|
||
|
|
||
| def disable_paddle_compat(): | ||
| """Turn OFF Paddle's torch-compat proxy if it is currently active. | ||
|
|
||
| Converted Paddle code injects ``paddle.enable_compat(level=2)``, which flips | ||
| process-global state: it installs an ``import torch`` -> Paddle proxy and | ||
| aliases ``paddle.*`` to the torch-aligned ``paddle.compat.*`` APIs. That state | ||
| must be cleared so it cannot leak into (a) a later test, or (b) a torch | ||
| *reference* run within the same test, whose ``import torch`` would otherwise be | ||
| proxied to Paddle so the reference would no longer be real torch. | ||
|
|
||
| Lazy and best-effort: a no-op when Paddle was never imported, so it does not | ||
| force a Paddle import (and thus does not change torch/paddle import ordering) | ||
| for tests that never touch Paddle. | ||
| """ | ||
| if "paddle" not in sys.modules: | ||
| return | ||
| paddle = sys.modules["paddle"] | ||
| try: | ||
| from paddle.compat.proxy import TORCH_PROXY_FINDER | ||
|
|
||
| while TORCH_PROXY_FINDER in sys.meta_path: | ||
| paddle.disable_compat() | ||
| except Exception: | ||
| pass | ||
|
|
||
|
|
||
| @pytest.fixture(autouse=True) | ||
| def _reset_paddle_compat_mode(): | ||
| """Disable torch-compat after every test so it cannot leak into a later one.""" | ||
| yield | ||
| disable_paddle_compat() | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
简化下代码,看有无更简单的写法,是否只需要修改visit_Module就可以?