diff --git a/.gitignore b/.gitignore index 457430299..a0e1213a6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ a.out hk test/*.pdf examples/*.pdf +examples/doubleMach.msh *.dot diff --git a/conda-env.yml b/conda-env.yml index 898623887..3a4d5b545 100644 --- a/conda-env.yml +++ b/conda-env.yml @@ -11,8 +11,7 @@ dependencies: - gh - mypy - conda-forge::numpy -# See https://github.com/illinois-ceesd/mirgecom/pull/1055: -- pocl=5.0=*_6 +- pocl - nvtop - clinfo - pudb diff --git a/examples/doubleMach.geo b/examples/doubleMach.geo new file mode 100644 index 000000000..3179f553a --- /dev/null +++ b/examples/doubleMach.geo @@ -0,0 +1,24 @@ +x0=1.0/6.0; +setsize=0.025; + +Point(1) = {0, 0, 0, setsize}; +Point(2) = {x0,0, 0, setsize}; +Point(3) = {4, 0, 0, setsize}; +Point(4) = {4, 1, 0, setsize}; +Point(5) = {0, 1, 0, setsize}; + +Line(1) = {1, 2}; +Line(2) = {2, 3}; +Line(5) = {3, 4}; +Line(6) = {4, 5}; +Line(7) = {5, 1}; + +Line Loop(8) = {-5, -6, -7, -1, -2}; +Plane Surface(8) = {8}; + +Physical Surface('domain') = {8}; +Physical Curve('ic1') = {6}; +Physical Curve('ic2') = {7}; +Physical Curve('ic3') = {1}; +Physical Curve('wall') = {2}; +Physical Curve('out') = {5}; diff --git a/examples/doublemach.py b/examples/doublemach.py index b5c9df3fe..4c2f39563 100644 --- a/examples/doublemach.py +++ b/examples/doublemach.py @@ -79,35 +79,16 @@ def get_doublemach_mesh(): from meshmode.mesh.io import ( read_gmsh, generate_gmsh, - ScriptSource, + FileSource, ) import os + local_path = os.path.dirname(os.path.abspath(__file__)) + geofile = os.path.join(local_path, "doubleMach.geo") meshfile = "doubleMach.msh" if not os.path.exists(meshfile): mesh = generate_gmsh( - ScriptSource(""" - x0=1.0/6.0; - setsize=0.025; - Point(1) = {0, 0, 0, setsize}; - Point(2) = {x0,0, 0, setsize}; - Point(3) = {4, 0, 0, setsize}; - Point(4) = {4, 1, 0, setsize}; - Point(5) = {0, 1, 0, setsize}; - Line(1) = {1, 2}; - Line(2) = {2, 3}; - Line(5) = {3, 4}; - Line(6) = {4, 5}; - Line(7) = {5, 1}; - Line Loop(8) = {-5, -6, -7, -1, -2}; - Plane Surface(8) = {8}; - Physical Surface('domain') = {8}; - Physical Curve('ic1') = {6}; - Physical Curve('ic2') = {7}; - Physical Curve('ic3') = {1}; - Physical Curve('wall') = {2}; - Physical Curve('out') = {5}; - """, "geo"), force_ambient_dim=2, dimensions=2, target_unit="M", - output_file_name=meshfile) + FileSource(geofile), force_ambient_dim=2, dimensions=2, target_unit="M", + output_file_path=meshfile) else: mesh = read_gmsh(meshfile, force_ambient_dim=2) diff --git a/examples/doublemach_physical_av.py b/examples/doublemach_physical_av.py index fac0879fd..b14f6b766 100644 --- a/examples/doublemach_physical_av.py +++ b/examples/doublemach_physical_av.py @@ -85,33 +85,16 @@ def get_doublemach_mesh(): from meshmode.mesh.io import ( read_gmsh, generate_gmsh, - ScriptSource, + FileSource, ) import os + local_path = os.path.dirname(os.path.abspath(__file__)) + geofile = os.path.join(local_path, "doubleMach.geo") meshfile = "doubleMach.msh" if not os.path.exists(meshfile): mesh = generate_gmsh( - ScriptSource(""" - x0=1.0/6.0; - setsize=0.025; - Point(1) = {0, 0, 0, setsize}; - Point(2) = {x0,0, 0, setsize}; - Point(3) = {4, 0, 0, setsize}; - Point(4) = {4, 1, 0, setsize}; - Point(5) = {0, 1, 0, setsize}; - Line(1) = {1, 2}; - Line(2) = {2, 3}; - Line(5) = {3, 4}; - Line(6) = {4, 5}; - Line(7) = {5, 1}; - Line Loop(8) = {-5, -6, -7, -1, -2}; - Plane Surface(8) = {8}; - Physical Surface('domain') = {8}; - Physical Curve('flow') = {1, 6, 7}; - Physical Curve('wall') = {2}; - Physical Curve('out') = {5}; - """, "geo"), force_ambient_dim=2, dimensions=2, target_unit="M", - output_file_name=meshfile) + FileSource(geofile), force_ambient_dim=2, dimensions=2, target_unit="M", + output_file_path=meshfile) else: mesh = read_gmsh(meshfile, force_ambient_dim=2) @@ -343,7 +326,9 @@ def _boundary_state(dcoll, dd_bdry, gas_model, state_minus, **kwargs): boundary_state_func=_boundary_state) boundaries = { - BoundaryDomainTag("flow"): flow_boundary, + BoundaryDomainTag("ic1"): flow_boundary, + BoundaryDomainTag("ic2"): flow_boundary, + BoundaryDomainTag("ic3"): flow_boundary, BoundaryDomainTag("wall"): AdiabaticSlipBoundary(), BoundaryDomainTag("out"): PressureOutflowBoundary(boundary_pressure=1.0), } diff --git a/mirgecom/array_context.py b/mirgecom/array_context.py index 423244c2f..1e0a9e0f9 100644 --- a/mirgecom/array_context.py +++ b/mirgecom/array_context.py @@ -239,6 +239,31 @@ def _check_gpu_oversubscription(actx: ArrayContext) -> None: f"Duplicate PCIe IDs: {dup}.") +def _check_pocl_svm_issue(actx: ArrayContext) -> None: + """Check for pocl version >= 6 and warn about SVM performance issue.""" + assert isinstance(actx, (PyOpenCLArrayContext, PytatoPyOpenCLArrayContext)) + + dev = actx.queue.device + + # Only check if running on an NVIDIA device, where SVM is required. Otherwise, + # just disable SVM in simutil.get_reasonable_memory_pool + from pyopencl.characterize import nv_compute_capability + if nv_compute_capability(dev) is None: + return + + # TODO: Check if this is still an issue in pocl7 + from pyopencl.characterize import get_pocl_version + pocl_version = get_pocl_version(dev.platform) + if pocl_version is not None and pocl_version[0] >= 6: + from warnings import warn + warn( + "Performance may be degraded on this device due to an issue with " + "shared virtual memory (SVM) in pocl version >= 6. If running " + "performance-critical simulations, consider " + "downgrading to version 5. See " + "https://github.com/illinois-ceesd/mirgecom/pull/1055 for more details.") + + def log_disk_cache_config(actx: ArrayContext) -> None: """Log the disk cache configuration.""" assert isinstance(actx, (PyOpenCLArrayContext, PytatoPyOpenCLArrayContext)) @@ -354,6 +379,7 @@ def initialize_actx( if actx_class_is_pyopencl(actx_class): _check_gpu_oversubscription(actx) _check_cache_dirs_node(actx) + _check_pocl_svm_issue(actx) log_disk_cache_config(actx) return actx diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 5f0fd3f97..4f7460b1b 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1859,7 +1859,19 @@ def get_reasonable_memory_pool(ctx: cl.Context, queue: cl.CommandQueue, pools over direct allocations. """ import pyopencl.tools as cl_tools - from pyopencl.characterize import has_coarse_grain_buffer_svm + from pyopencl.characterize import ( + get_pocl_version, + has_coarse_grain_buffer_svm, + nv_compute_capability, + ) + + # Avoid SVM slowdown in pocl6 (except on NVIDIA where SVM is mandatory) + # https://github.com/illinois-ceesd/mirgecom/pull/1055 + # TODO: Check if this is still an issue in pocl7 + if nv_compute_capability(queue.device) is None: + pocl_version = get_pocl_version(queue.device.platform) + if pocl_version is not None and pocl_version[0] >= 6: + force_buffer = True if force_buffer and force_non_pool: logger.info(f"Using non-pooled CL buffer allocations on {queue.device}.") diff --git a/requirements.txt b/requirements.txt index b078bc31e..c6629a369 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ git+https://github.com/pythological/kanren.git#egg=miniKanren --editable git+https://github.com/inducer/dagrt.git#egg=dagrt --editable git+https://github.com/inducer/leap.git#egg=leap --editable git+https://github.com/inducer/modepy.git#egg=modepy +--editable git+https://github.com/inducer/pytools.git#egg=pytools --editable git+https://github.com/illinois-ceesd/arraycontext.git@production#egg=arraycontext --editable git+https://github.com/illinois-ceesd/meshmode.git@production#egg=meshmode --editable git+https://github.com/illinois-ceesd/grudge.git@production#egg=grudge diff --git a/test/test_eos.py b/test/test_eos.py index 390748490..e2e7dc388 100644 --- a/test/test_eos.py +++ b/test/test_eos.py @@ -445,7 +445,7 @@ def inf_norm(x): # print(f"pyro_eos.temp = {temperature}") # print(f"pyro_eos.e = {internal_energy}") - tol = 5e-14 + tol = 6e-14 assert inf_norm((cv.mass - pyro_rho) / pyro_rho) < tol assert inf_norm((temperature - pyro_t) / pyro_t) < tol assert inf_norm((internal_energy - pyro_e) / pyro_e) < tol diff --git a/test/test_overintegration.py b/test/test_overintegration.py index 2365c5306..00749e50e 100644 --- a/test/test_overintegration.py +++ b/test/test_overintegration.py @@ -326,6 +326,13 @@ def test_dealiasing_with_overintegration(ctx_factory, tpe, warp): q_n = 2*poly_degree + warp - 1 if tpe: q_n = max(poly_degree, q_expected - 2) + + if q_n == poly_degree: + pytest.xfail( + "Doesn't currently work when quadrature discr has same number of DOFs " + "as base due to bug in loop fusion. See " + "https://github.com/inducer/meshmode/issues/453.") + l2_error_n = run_agitator(ctx_factory=ctx_factory, use_overintegration=True, p_adv=warp, order=poly_degree, tpe=tpe,