From 8e7ddda588e51c4ee339d169e57a6750e882e1fe Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Wed, 28 Oct 2020 18:22:51 -0500 Subject: [PATCH 001/385] Initial implementation of artificial viscosity for the euler equations --- mirgecom/artificial_viscosity.py | 124 +++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 mirgecom/artificial_viscosity.py diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py new file mode 100644 index 000000000..c496e3a1e --- /dev/null +++ b/mirgecom/artificial_viscosity.py @@ -0,0 +1,124 @@ +r""":mod:`mirgecom.artificial viscosity` applys and artifical viscosity to the euler equations +""" + +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +#from dataclasses import dataclass + +import numpy as np +from pytools.obj_array import make_obj_array +from meshmode.dof_array import thaw +from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa +from grudge.eager import ( + interior_trace_pair, + cross_rank_trace_pairs +) +from grudge.symbolic.primitives import TracePair +from mirgecom.euler import split_conserved, join_conserved + + +def scalar(s): + """Create an object array for a scalar.""" + return make_obj_array([s]) + + +def dissapative_flux(discr, q): + + dim = discr.dim + cv = split_conserved(dim, q) + + return join_conserved(dim, + mass=np.ones(dim)*scalar(cv.mass), + energy=np.ones(dim)*scalar(cv.energy), + momentum= np.ones((dim,dim))*cv.momentum ) + +def _facial_flux(discr, q_tpair): + + dim = discr.dim + + actx = q_tpair[0].int.array_context + + flux_int = dissapative_flux(discr,q_tpair.int); + flux_ext = dissapative_flux(discr,q_tpair.ext); + flux_dis = 0.5*(flux_ext + flux_int); + + normal = thaw(actx, discr.normal(q_tpair.dd)) + + flux_out = flux_dis @ normal + + return discr.project(q_tpair.dd, "all_faces", flux_out) + +def artificial_viscosity(discr, r): + r"""Compute artifical viscosity for the euler equations + + """ + + #compute dissapation flux + vol_flux_r = dissapative_flux(discr, r) + dflux_r = discr.weak_div(vol_flux_r) + + #interior face flux + iff_r = _facial_flux(discr, q_tpair=interior_trace_pair(discr,r)) + + #partition boundaries flux + pbf_r = sum( + _facial_flux(discr, q_tpair=part_pair) + for part_pair in cross_rank_trace_pairs(discr, r) + ) + + #domain boundary flux + dir_r = discr.project("vol", BTAG_ALL,r) + dbf_r = _facial_flux( + discr, + q_tpair=TracePair(BTAG_ALL,interior=dir_r,exterior=dir_r) + ) + + q = discr.inverse_mass( 1.0e-2 * (dflux_r - discr.face_mass(iff_r + pbf_r + dbf_r))) + + #flux of q + vol_flux_q = dissapative_flux(discr, q) + dflux_q = discr.weak_div(vol_flux_q) + + #interior face flux of q + iff_q = _facial_flux(discr, q_tpair=interior_trace_pair(discr,q)) + + #flux across partition boundaries + pbf_q = sum( + _facial_flux(discr, q_tpair=part_pair) + for part_pair in cross_rank_trace_pairs(discr, q) + ) + + #dombain boundary flux + dir_q = discr.project("vol",BTAG_ALL, q); + dbf_q = _facial_flux( + discr, + q_tpair=TracePair(BTAG_ALL,interior=dir_q,exterior=dir_q) + ) + + + return discr.inverse_mass( + dflux_q - discr.face_mass(iff_q + pbf_q + dbf_q) + ) + From 59f80c34735c3cf1638241aaf05bcefb3f4e395f Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Sat, 31 Oct 2020 19:55:24 -0500 Subject: [PATCH 002/385] Initial implementation of smoothness indicator --- mirgecom/tag_cells.py | 103 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 mirgecom/tag_cells.py diff --git a/mirgecom/tag_cells.py b/mirgecom/tag_cells.py new file mode 100644 index 000000000..98fc21842 --- /dev/null +++ b/mirgecom/tag_cells.py @@ -0,0 +1,103 @@ +r""":mod:`mirgecom.tag_cells` Computes smoothness indicator + +Perssons smoothness indicator: + +.. math:: + + S_e = \frac{\langle u_{N_p} - u_{N_{p-1}}, u_{N_p} - u_{N_{p-1}}\rangle_e}{\langle u_{N_p}, u_{N_p} \rangle_e} + +""" +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import numpy as np +import loopy as lp +from grudge import sym +from meshmode.dof_array import DOFArray +from modepy import vandermonde +from pytools import memoize_in + + +def linear_operator_kernel(): + """Apply linear operator to all elements.""" + from meshmode.array_context import make_loopy_program + knl = make_loopy_program( + """{[iel,idof,j]: + 0<=iel Date: Mon, 2 Nov 2020 23:13:43 -0600 Subject: [PATCH 003/385] Updated modal smoothness indicator to mode_ids to select highest mode --- mirgecom/tag_cells.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/mirgecom/tag_cells.py b/mirgecom/tag_cells.py index 98fc21842..bf7e196d6 100644 --- a/mirgecom/tag_cells.py +++ b/mirgecom/tag_cells.py @@ -37,6 +37,7 @@ from meshmode.dof_array import DOFArray from modepy import vandermonde from pytools import memoize_in +from pytools.obj_array import make_obj_array def linear_operator_kernel(): @@ -52,7 +53,6 @@ def linear_operator_kernel(): knl = lp.tag_array_axes(knl, "mat", "stride:auto,stride:auto") return knl -#This is hardcoded for order=3 elements currently, working on generalizing def compute_smoothness_indicator(): """Compute the smoothness indicator for all elements.""" from meshmode.array_context import make_loopy_program @@ -61,8 +61,8 @@ def compute_smoothness_indicator(): 0<=iel Date: Fri, 6 Nov 2020 15:33:28 -0600 Subject: [PATCH 004/385] Fixed updated artificial viscosity implementation to use desired equations --- mirgecom/artificial_viscosity.py | 165 +++++++++++++++++++++---------- 1 file changed, 112 insertions(+), 53 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index c496e3a1e..20eb6eccb 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -28,7 +28,7 @@ #from dataclasses import dataclass import numpy as np -from pytools.obj_array import make_obj_array +from pytools.obj_array import make_obj_array, obj_array_vectorize, flat_obj_array,obj_array_vectorize_n_args from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.eager import ( @@ -39,86 +39,145 @@ from mirgecom.euler import split_conserved, join_conserved -def scalar(s): - """Create an object array for a scalar.""" - return make_obj_array([s]) +def _facial_flux_r(discr, q_tpair): + dim = discr.dim + actx = q_tpair[0].int.array_context -def dissapative_flux(discr, q): + flux_dis = q_tpair.avg - dim = discr.dim - cv = split_conserved(dim, q) + normal = thaw(actx, discr.normal(q_tpair.dd)) - return join_conserved(dim, - mass=np.ones(dim)*scalar(cv.mass), - energy=np.ones(dim)*scalar(cv.energy), - momentum= np.ones((dim,dim))*cv.momentum ) + flux_out = flux_dis * normal + + # Can't do it here... "obj arrays not allowed on compute device" + #def flux_calc(flux): + # return (flux * normal) + #flux_out = obj_array_vectorize(flux_calc,flux_dis) -def _facial_flux(discr, q_tpair): + return discr.project(q_tpair.dd, "all_faces", flux_out) + +def _facial_flux_q(discr, q_tpair): dim = discr.dim actx = q_tpair[0].int.array_context - flux_int = dissapative_flux(discr,q_tpair.int); - flux_ext = dissapative_flux(discr,q_tpair.ext); - flux_dis = 0.5*(flux_ext + flux_int); - normal = thaw(actx, discr.normal(q_tpair.dd)) - flux_out = flux_dis @ normal + flux_out = np.dot(q_tpair.avg,normal) return discr.project(q_tpair.dd, "all_faces", flux_out) -def artificial_viscosity(discr, r): +def artificial_viscosity(discr, t, eos, boundaries, r, alpha): r"""Compute artifical viscosity for the euler equations """ - #compute dissapation flux - vol_flux_r = dissapative_flux(discr, r) - dflux_r = discr.weak_div(vol_flux_r) + #Cannot call weak_grad on obj of nd arrays, use obj_array_vectorize as work around + dflux_r = obj_array_vectorize(discr.weak_grad,r) + #interior face flux - iff_r = _facial_flux(discr, q_tpair=interior_trace_pair(discr,r)) - #partition boundaries flux - pbf_r = sum( - _facial_flux(discr, q_tpair=part_pair) - for part_pair in cross_rank_trace_pairs(discr, r) - ) + #Doesn't work: something related to obj on compute device + #Not 100% on reason + #qin = interior_trace_pair(discr,r) + #iff_r = _facial_flux(discr,q_tpair=qin) - #domain boundary flux - dir_r = discr.project("vol", BTAG_ALL,r) - dbf_r = _facial_flux( - discr, - q_tpair=TracePair(BTAG_ALL,interior=dir_r,exterior=dir_r) - ) + #Work around? + def my_facialflux_r_interior(q): + qin = interior_trace_pair(discr,make_obj_array([q])) + return _facial_flux_r(discr,q_tpair=qin) - q = discr.inverse_mass( 1.0e-2 * (dflux_r - discr.face_mass(iff_r + pbf_r + dbf_r))) + iff_r = obj_array_vectorize(my_facialflux_r_interior,r) + + + #partition boundaries flux + #flux across partition boundaries + def my_facialflux_r_partition(q): + qin = cross_rank_trace_pairs(discr,q) + return sum(_facial_flux_r(discr,q_tpair=part_pair) for part_pair in cross_rank_trace_pairs(discr,make_obj_array([q])) ) + + pbf_r = obj_array_vectorize(my_facialflux_r_partition,r) + + #pbf_r = sum( + # _facial_flux_r(discr, q_tpair=part_pair) + # for part_pair in cross_rank_trace_pairs(discr, r) + #) + + #domain boundary flux basic boundary implementation + #def my_facialflux2(r): + # dir_r = discr.project("vol", BTAG_ALL,make_obj_array([r])) + # dbf_r = _facial_flux( + # discr, + # q_tpair=TracePair(BTAG_ALL,interior=dir_r,exterior=dir_r) + # ) + # return (dbf_r) + #dbf_r = obj_array_vectorize(my_facialflux2,r) + + + #True boundary implementation + #Okay, not sure about this... + #What I am attempting: + # 1. Loop through all the boundaries + # 2. Define a function my_TP that performes the trace pair for the given boundary + # given a solution variable + # 3. Get the external solution from the boundary routine + # 4. Get hte projected internal solution + # 5. Compute the boundary flux as a sum over boundaries, using the obj_array_vectorize to + # pass each solution variable one at a time + # DO I really need to do this like this? + dbf_r = 0.0*iff_r + for btag in boundaries: + def my_facialflux_r_boundary(sol_ext,sol_int): + q_tpair = TracePair(btag,interior=make_obj_array([sol_int]),exterior=make_obj_array([sol_ext])) + return _facial_flux_r(discr,q_tpair=q_tpair) + r_ext=boundaries[btag].interior_sol(discr,eos=eos,btag=btag,t=t,q=r) + r_int=discr.project("vol",btag,r) + dbf_r = dbf_r + obj_array_vectorize_n_args(my_facialflux_r_boundary,r_ext,r_int) + + + #Compute q, half way done! + q = discr.inverse_mass( -alpha * (dflux_r - discr.face_mass(iff_r + pbf_r + dbf_r))) #flux of q - vol_flux_q = dissapative_flux(discr, q) - dflux_q = discr.weak_div(vol_flux_q) + + #Again we need to vectorize + #q is a object array of object arrays (dim,) of DOFArrays (?) + dflux_q = obj_array_vectorize(discr.weak_div,q) #interior face flux of q - iff_q = _facial_flux(discr, q_tpair=interior_trace_pair(discr,q)) + def my_facialflux_q_interior(q): + qin = interior_trace_pair(discr,q) + iff_q = _facial_flux_q(discr, q_tpair=qin) + return (iff_q) - #flux across partition boundaries - pbf_q = sum( - _facial_flux(discr, q_tpair=part_pair) - for part_pair in cross_rank_trace_pairs(discr, q) - ) - - #dombain boundary flux - dir_q = discr.project("vol",BTAG_ALL, q); - dbf_q = _facial_flux( - discr, - q_tpair=TracePair(BTAG_ALL,interior=dir_q,exterior=dir_q) - ) - + iff_q = obj_array_vectorize(my_facialflux_q_interior,q) - return discr.inverse_mass( - dflux_q - discr.face_mass(iff_q + pbf_q + dbf_q) - ) + #flux across partition boundaries + def my_facialflux_q_partition(q): + qin = cross_rank_trace_pairs(discr,q) + return sum(_facial_flux_q(discr,q_tpair=part_pair) for part_pair in cross_rank_trace_pairs(discr,make_obj_array([q])) ) + + pbf_q = obj_array_vectorize(my_facialflux_q_partition,q) + #pbf_q = sum( + # _facial_flux_q(discr, q_tpair=part_pair) + # for part_pair in cross_rank_trace_pairs(discr, q) + #) + + def my_facialflux_q_boundary(q): + #dombain boundary flux + dir_q = discr.project("vol",BTAG_ALL, q); + dbf_q = _facial_flux_q( + discr, + q_tpair=TracePair(BTAG_ALL,interior=dir_q,exterior=dir_q) + ) + return(dbf_q) + + dbf_q = obj_array_vectorize(my_facialflux_q_boundary,q) + + #Return the rhs contribution + return ( discr.inverse_mass( -dflux_q + discr.face_mass(iff_q + pbf_q + dbf_q) ) ) + From 822d86394aa88dc04cf47d1991a3d07bffff923c Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Fri, 6 Nov 2020 15:41:13 -0600 Subject: [PATCH 005/385] Add boundary condition function to return projected boundary solution --- mirgecom/artificial_viscosity.py | 2 +- mirgecom/boundary.py | 143 +++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 1 deletion(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 20eb6eccb..7711cd7fc 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -133,7 +133,7 @@ def my_facialflux_r_partition(q): def my_facialflux_r_boundary(sol_ext,sol_int): q_tpair = TracePair(btag,interior=make_obj_array([sol_int]),exterior=make_obj_array([sol_ext])) return _facial_flux_r(discr,q_tpair=q_tpair) - r_ext=boundaries[btag].interior_sol(discr,eos=eos,btag=btag,t=t,q=r) + r_ext=boundaries[btag].exterior_sol(discr,eos=eos,btag=btag,t=t,q=r) r_int=discr.project("vol",btag,r) dbf_r = dbf_r + obj_array_vectorize_n_args(my_facialflux_r_boundary,r_ext,r_int) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index c867f94c1..86fe509d2 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -5,6 +5,7 @@ .. autoclass:: PrescribedBoundary .. autoclass:: DummyBoundary +.. autoclass:: AdiabaticSlipBoundary """ __copyright__ = """ @@ -31,10 +32,14 @@ THE SOFTWARE. """ +import numpy as np +from pytools.obj_array import make_obj_array from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from mirgecom.eos import IdealSingleGas +# from mirgecom.euler import split_conserved from grudge.symbolic.primitives import TracePair +from mirgecom.euler import split_conserved, join_conserved class PrescribedBoundary: @@ -67,6 +72,17 @@ def boundary_pair( int_soln = discr.project("vol", btag, q) return TracePair(btag, interior=int_soln, exterior=ext_soln) + def exterior_sol( + self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() + ): + """Get the interior solution on the boundary.""" + actx = q[0].array_context + + boundary_discr = discr.discr_from_dd(btag) + nodes = thaw(actx, boundary_discr.nodes()) + ext_soln = self._userfunc(t, nodes) + return ext_soln + class DummyBoundary: """Use the boundary-adjacent solution as the boundary solution. @@ -74,9 +90,136 @@ class DummyBoundary: .. automethod:: boundary_pair """ + def __init__(self): + """Initialize dummy boundary.""" + self.test = 1 + def boundary_pair( self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() ): """Get the interior and exterior solution on the boundary.""" dir_soln = discr.project("vol", btag, q) return TracePair(btag, interior=dir_soln, exterior=dir_soln) + + def exterior_sol( + self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() + ): + """Get the interior and exterior solution on the boundary.""" + dir_soln = discr.project("vol", btag, q) + return dir_soln + +class AdiabaticSlipBoundary: + """Adiabatic slip boundary for inviscid flows. + + a.k.a. Reflective inviscid wall boundary + + This class implements an adiabatic reflective slip boundary + wherein the normal component of velocity at the wall is 0, and + tangential components are preserved. These perfectly reflecting + conditions are used by the forward-facing step case in + JSH/TW Nodal DG Methods, Section 6.6 DOI: 10.1007/978-0-387-72067-8 and + described in detail by Poinsot and Lele's JCP paper + http://acoustics.ae.illinois.edu/pdfs/poinsot-lele-1992.pdf + + .. automethod:: boundary_pair + """ + + def __init__(self): + """Create the adiabatic slip boundary.""" + self.test = 1 + + def boundary_pair( + self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() + ): + """Get the interior and exterior solution on the boundary. + + The exterior solution is set such that there will be vanishing + flux through the boundary, preserving mass, momentum (magnitude) and + energy. + rho_plus = rho_minus + v_plus = v_minus - 2 * (v_minus . n_hat) * n_hat + mom_plus = rho_plus * v_plus + E_plus = E_minus + """ + # Grab some boundary-relevant data + dim = discr.dim + cv = split_conserved(dim, q) + actx = cv.mass.array_context + + # Grab a unit normal to the boundary + normal = thaw(actx, discr.normal(btag)) + normal_mag = actx.np.sqrt(np.dot(normal, normal)) + nhat_mult = 1.0 / normal_mag + nhat = normal * make_obj_array([nhat_mult]) + + # Get the interior/exterior solns + int_soln = discr.project("vol", btag, q) + bndry_cv = split_conserved(dim, int_soln) + # bpressure = eos.pressure(bndry_cv) + + # Subtract out the 2*wall-normal component + # of velocity from the velocity at the wall to + # induce an equal but opposite wall-normal (reflected) wave + # preserving the tangential component + wall_velocity = bndry_cv.momentum / make_obj_array([bndry_cv.mass]) + nvel_comp = np.dot(wall_velocity, nhat) # part of velocity normal to wall + wnorm_vel = nhat * make_obj_array([nvel_comp]) # wall-normal velocity vec + wall_velocity = wall_velocity - 2.0 * wnorm_vel # prescribed ext velocity + + # Re-calculate the boundary solution with the new + # momentum + bndry_cv.momentum = wall_velocity * make_obj_array([bndry_cv.mass]) + # bndry_cv.energy = eos.total_energy(bndry_cv, bpressure) + bndry_soln = join_conserved(dim=dim, mass=bndry_cv.mass, + energy=bndry_cv.energy, + momentum=bndry_cv.momentum) + + return TracePair(btag, interior=int_soln, exterior=bndry_soln) + + def exterior_sol( + self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() + ): + """Get the interior and exterior solution on the boundary. + + The exterior solution is set such that there will be vanishing + flux through the boundary, preserving mass, momentum (magnitude) and + energy. + rho_plus = rho_minus + v_plus = v_minus - 2 * (v_minus . n_hat) * n_hat + mom_plus = rho_plus * v_plus + E_plus = E_minus + """ + # Grab some boundary-relevant data + dim = discr.dim + cv = split_conserved(dim, q) + actx = cv.mass.array_context + + # Grab a unit normal to the boundary + normal = thaw(actx, discr.normal(btag)) + normal_mag = actx.np.sqrt(np.dot(normal, normal)) + nhat_mult = 1.0 / normal_mag + nhat = normal * make_obj_array([nhat_mult]) + + # Get the interior/exterior solns + int_soln = discr.project("vol", btag, q) + bndry_cv = split_conserved(dim, int_soln) + # bpressure = eos.pressure(bndry_cv) + + # Subtract out the 2*wall-normal component + # of velocity from the velocity at the wall to + # induce an equal but opposite wall-normal (reflected) wave + # preserving the tangential component + wall_velocity = bndry_cv.momentum / make_obj_array([bndry_cv.mass]) + nvel_comp = np.dot(wall_velocity, nhat) # part of velocity normal to wall + wnorm_vel = nhat * make_obj_array([nvel_comp]) # wall-normal velocity vec + wall_velocity = wall_velocity - 2.0 * wnorm_vel # prescribed ext velocity + + # Re-calculate the boundary solution with the new + # momentum + bndry_cv.momentum = wall_velocity * make_obj_array([bndry_cv.mass]) + # bndry_cv.energy = eos.total_energy(bndry_cv, bpressure) + bndry_soln = join_conserved(dim=dim, mass=bndry_cv.mass, + energy=bndry_cv.energy, + momentum=bndry_cv.momentum) + + return bndry_soln From e0f0ce43e1c76017aa301827ab854890b81ed474 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Fri, 6 Nov 2020 15:41:58 -0600 Subject: [PATCH 006/385] Added double mach reflection example problem --- examples/doublemach-mpi.py | 181 +++++++++++++++++++++++++++++++++++++ mirgecom/initializers.py | 115 +++++++++++++++++++++++ 2 files changed, 296 insertions(+) create mode 100644 examples/doublemach-mpi.py diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py new file mode 100644 index 000000000..be32f4ac5 --- /dev/null +++ b/examples/doublemach-mpi.py @@ -0,0 +1,181 @@ +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" +import logging +import pyopencl as cl +import pyopencl.tools as cl_tools +from functools import partial + +from meshmode.array_context import PyOpenCLArrayContext +from meshmode.dof_array import thaw +from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa +from grudge.eager import EagerDGDiscretization +from grudge.shortcuts import make_visualizer + + +from mirgecom.euler import inviscid_operator +from mirgecom.artificial_viscosity import artificial_viscosity +from mirgecom.heat import heat_operator +from mirgecom.simutil import ( + inviscid_sim_timestep, + sim_checkpoint, + create_parallel_grid, + ExactSolutionMismatch, +) +from mirgecom.io import make_init_message +from mirgecom.mpi import mpi_entry_point + +from mirgecom.integrators import rk4_step +from mirgecom.steppers import advance_state +from mirgecom.boundary import AdiabaticSlipBoundary, PrescribedBoundary +from mirgecom.initializers import DoubleMachReflection, SodShock1D +from mirgecom.eos import IdealSingleGas + +from pytools.obj_array import obj_array_vectorize + +logger = logging.getLogger(__name__) + + +@mpi_entry_point +def main(ctx_factory=cl.create_some_context): + cl_ctx = ctx_factory() + queue = cl.CommandQueue(cl_ctx) + actx = PyOpenCLArrayContext(queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + dim = 2 + nel = (40,10) + order = 3 + # tolerate large errors; case is unstable + exittol = 2.0 #.2 + t_final = 1.0 + current_cfl = 0.1 + current_dt = .00031250 + current_t = 0 + eos = IdealSingleGas() + initializer = DoubleMachReflection(dim) + #initializer = SodShock1D(dim,x0=0.5) + casename = "sod1d" + #boundaries = {BTAG_ALL: AdiabaticSlipBoundary()} + from grudge import sym + boundaries = {sym.DTAG_BOUNDARY("ic1"): PrescribedBoundary(initializer), + sym.DTAG_BOUNDARY("ic2"): PrescribedBoundary(initializer), + sym.DTAG_BOUNDARY("ic3"): PrescribedBoundary(initializer), + sym.DTAG_BOUNDARY("wall"): AdiabaticSlipBoundary(), + sym.DTAG_BOUNDARY("out"): AdiabaticSlipBoundary()} + constant_cfl = False + nstatus = 10 + nviz = 25 + rank = 0 + checkpoint_t = current_t + current_step = 0 + timestepper = rk4_step + box_ll = (0.0,0.0) + box_ur = (4.0,1.0) + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + + #from meshmode.mesh.generation import generate_regular_rect_mesh + #generate_grid = partial(generate_regular_rect_mesh, a=box_ll, + # b=box_ur, n=nel ) + #local_mesh, global_nelements = create_parallel_grid(comm, generate_grid) + + + from meshmode.mesh.io import read_gmsh, generate_gmsh, ScriptWithFilesSource + local_mesh = read_gmsh("doubleMach1.msh",force_ambient_dim=2) + global_nelements = local_mesh.nelements + + local_nelements = local_mesh.nelements + + discr = EagerDGDiscretization( + actx, local_mesh, order=order, mpi_communicator=comm + ) + nodes = thaw(actx, discr.nodes()) + current_state = initializer(0, nodes) + + #visualizer = make_visualizer(discr, discr.order + 3 + # if discr.dim == 2 else discr.order) + visualizer = make_visualizer(discr, discr.order + if discr.dim == 2 else discr.order) + initname = initializer.__class__.__name__ + eosname = eos.__class__.__name__ + init_message = make_init_message(dim=dim, order=order, + nelements=local_nelements, + global_nelements=global_nelements, + dt=current_dt, t_final=t_final, nstatus=nstatus, + nviz=nviz, cfl=current_cfl, + constant_cfl=constant_cfl, initname=initname, + eosname=eosname, casename=casename) + if rank == 0: + logger.info(init_message) + + get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, + dt=current_dt, cfl=current_cfl, eos=eos, + t_final=t_final, constant_cfl=constant_cfl) + def my_av(state): + return (heat_operator(discr,alpha=1.0e-3,w=state) ) + + def my_rhs(t, state): + return ( + inviscid_operator(discr, q=state, t=t,boundaries=boundaries, eos=eos) + + artificial_viscosity(discr,t=t, r=state, eos=eos, boundaries=boundaries, alpha=4.0e-2) + ) + #return ( + # inviscid_operator(discr, q=state, t=t,boundaries=boundaries, eos=eos) + # + artificial_viscosity(discr, r=state, eos=eos, boundaries=boundaries, alpha=1.0e-3) + + def my_checkpoint(step, t, dt, state): + return sim_checkpoint(discr, visualizer, eos, q=state, + vizname=casename, step=step, + t=t, dt=dt, nstatus=nstatus, nviz=nviz, + exittol=exittol, constant_cfl=constant_cfl, comm=comm) + + try: + (current_step, current_t, current_state) = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + checkpoint=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final) + except ExactSolutionMismatch as ex: + current_step = ex.step + current_t = ex.t + current_state = ex.state + + # if current_t != checkpoint_t: + if rank == 0: + logger.info("Checkpointing final state ...") + my_checkpoint(current_step, t=current_t, + dt=(current_t - checkpoint_t), + state=current_state) + + if current_t - t_final < 0: + raise ValueError("Simulation exited abnormally") + + +if __name__ == "__main__": + logging.basicConfig(format="%(message)s", level=logging.INFO) + main() + +# vim: foldmethod=marker diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index e873ae117..a28ce7187 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -5,6 +5,7 @@ ^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: Vortex2D .. autoclass:: SodShock1D +.. autoclass:: DoubleMachReflection .. autoclass:: Lump .. autoclass:: Uniform """ @@ -212,6 +213,120 @@ def __call__(self, t, x_vec, eos=IdealSingleGas()): return flat_obj_array(mass, energy, mom) +class DoubleMachReflection: + r"""Implement the double shock reflection problem + + - Woodward and Collela + + The inital condition is defined + + .. math:: + + (\rho,u,v,P) = + + This function only serves as an initial condition + + .. automethod:: __init__ + .. automethod:: __call__ + """ + + def __init__( + self,dim=2, x0=1.0/6.0, us=4.0 + ): + """Initialize initial condition options + + Parameters + ---------- + dim: int + dimension of domain, must be 2 + x0: float + location of shock + us: float + shock speed + """ + self._x0 = x0 + self._dim = dim + self._us = us + + def __call__(self, t, x_vec, eos=IdealSingleGas()): + """ + Create the 1D Sod's shock solution at locations *x_vec*. + + Parameters + ---------- + t: float + Current time at which the solution is desired (unused) + x_vec: numpy.ndarray + Nodal coordinates + eos: :class:`mirgecom.eos.GasEOS` + Equation of state class to be used in construction of soln (if needed) + """ + assert self._dim == 2, "only defined for dim=2" + + gm1 = eos.gamma() - 1.0 + gmn1 = 1.0 / gm1 + x_rel = x_vec[0] + y_rel = x_vec[1] + actx = x_rel.array_context + + zeros = 0*x_rel + + x0 = zeros + self._x0 + us = zeros + self._us + t = zeros + t + + + #Mach 10 + #rhol = zeros + 8.0 + #rhor = zeros + 1.4 + + #ul = zeros + 8.25*np.cos(np.pi/6.0) + #ur = zeros + 0.0 + #vl = zeros - 8.25*np.sin(np.pi/6.0) + #vr = zeros + 0.0 + #rhoel = zeros + gmn1 * 116.5 + #rhoer = zeros + gmn1 * 1.0 + + #Mach 2.0 + #rhol = zeros + 2.666666*1.4 + #rhor = zeros + 1.4 + + #ul = zeros + 1.25*np.cos(np.pi/6.0) + #ur = zeros + 0.0 + #vl = zeros - 1.25*np.sin(np.pi/6.0) + #vr = zeros + 0.0 + #rhoel = zeros + gmn1 * 4.5 + #rhoer = zeros + gmn1 * 1.0 + + #Mach 4.0 + rhol = zeros + 4.57142857*1.4 + rhor = zeros + 1.4 + + ul = zeros + 3.125*np.cos(np.pi/6.0) + ur = zeros + 0.0 + vl = zeros - 3.125*np.sin(np.pi/6.0) + vr = zeros + 0.0 + rhoel = zeros + gmn1 * 18.5 + rhoer = zeros + gmn1 * 1.0 + + #yesno = x_rel > (x0 + y_rel/np.sqrt(3.0) + 2.0*us*t/np.sqrt(3.0)) + #mass = actx.np.where(yesno, rhor, rhol) + #rhoe = actx.np.where(yesno, rhoer, rhoel) + #u = actx.np.where(yesno, ur, ul) + #v = actx.np.where(yesno, vr, vl) + xinter = (x0 + y_rel/np.sqrt(3.0) + 2.0*us*t/np.sqrt(3.0)) + sigma=0.05 + xtanh = 1.0/sigma*(x_rel-xinter) + mass = rhol/2.0*(actx.np.tanh(-xtanh)+1.0)+rhor/2.0*(actx.np.tanh(xtanh)+1.0) + rhoe = rhoel/2.0*(actx.np.tanh(-xtanh)+1.0)+rhoer/2.0*(actx.np.tanh(xtanh)+1.0) + u = ul/2.0*(actx.np.tanh(-xtanh)+1.0)+ur/2.0*(actx.np.tanh(xtanh)+1.0) + v = vl/2.0*(actx.np.tanh(-xtanh)+1.0)+vr/2.0*(actx.np.tanh(xtanh)+1.0) + rhou = mass*u + rhov = mass*v + energy = rhoe + 0.5*mass*(u*u + v*v) + + return flat_obj_array(mass, energy, rhou, rhov) + class Lump: r"""Implement an N-dimensional Gaussian lump of mass. From d35771a069b10b87ccc7ae790932c058c6bfb14f Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Fri, 6 Nov 2020 15:41:58 -0600 Subject: [PATCH 007/385] Added double mach reflection example problem --- examples/doublemach-mpi.py | 180 +++++++++++++++++++++++++++++++++++++ mirgecom/initializers.py | 115 ++++++++++++++++++++++++ 2 files changed, 295 insertions(+) create mode 100644 examples/doublemach-mpi.py diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py new file mode 100644 index 000000000..6410e0e36 --- /dev/null +++ b/examples/doublemach-mpi.py @@ -0,0 +1,180 @@ +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" +import logging +import pyopencl as cl +import pyopencl.tools as cl_tools +from functools import partial + +from meshmode.array_context import PyOpenCLArrayContext +from meshmode.dof_array import thaw +from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa +from grudge.eager import EagerDGDiscretization +from grudge.shortcuts import make_visualizer + + +from mirgecom.euler import inviscid_operator +from mirgecom.artificial_viscosity import artificial_viscosity +from mirgecom.simutil import ( + inviscid_sim_timestep, + sim_checkpoint, + create_parallel_grid, + ExactSolutionMismatch, +) +from mirgecom.io import make_init_message +from mirgecom.mpi import mpi_entry_point + +from mirgecom.integrators import rk4_step +from mirgecom.steppers import advance_state +from mirgecom.boundary import AdiabaticSlipBoundary, PrescribedBoundary +from mirgecom.initializers import DoubleMachReflection, SodShock1D +from mirgecom.eos import IdealSingleGas + +from pytools.obj_array import obj_array_vectorize + +logger = logging.getLogger(__name__) + + +@mpi_entry_point +def main(ctx_factory=cl.create_some_context): + cl_ctx = ctx_factory() + queue = cl.CommandQueue(cl_ctx) + actx = PyOpenCLArrayContext(queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + dim = 2 + nel = (40,10) + order = 3 + # tolerate large errors; case is unstable + exittol = 2.0 #.2 + t_final = 1.0 + current_cfl = 0.1 + current_dt = .00031250 + current_t = 0 + eos = IdealSingleGas() + initializer = DoubleMachReflection(dim) + #initializer = SodShock1D(dim,x0=0.5) + casename = "sod1d" + #boundaries = {BTAG_ALL: AdiabaticSlipBoundary()} + from grudge import sym + boundaries = {sym.DTAG_BOUNDARY("ic1"): PrescribedBoundary(initializer), + sym.DTAG_BOUNDARY("ic2"): PrescribedBoundary(initializer), + sym.DTAG_BOUNDARY("ic3"): PrescribedBoundary(initializer), + sym.DTAG_BOUNDARY("wall"): AdiabaticSlipBoundary(), + sym.DTAG_BOUNDARY("out"): AdiabaticSlipBoundary()} + constant_cfl = False + nstatus = 10 + nviz = 25 + rank = 0 + checkpoint_t = current_t + current_step = 0 + timestepper = rk4_step + box_ll = (0.0,0.0) + box_ur = (4.0,1.0) + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + + #from meshmode.mesh.generation import generate_regular_rect_mesh + #generate_grid = partial(generate_regular_rect_mesh, a=box_ll, + # b=box_ur, n=nel ) + #local_mesh, global_nelements = create_parallel_grid(comm, generate_grid) + + + from meshmode.mesh.io import read_gmsh, generate_gmsh, ScriptWithFilesSource + local_mesh = read_gmsh("doubleMach1.msh",force_ambient_dim=2) + global_nelements = local_mesh.nelements + + local_nelements = local_mesh.nelements + + discr = EagerDGDiscretization( + actx, local_mesh, order=order, mpi_communicator=comm + ) + nodes = thaw(actx, discr.nodes()) + current_state = initializer(0, nodes) + + #visualizer = make_visualizer(discr, discr.order + 3 + # if discr.dim == 2 else discr.order) + visualizer = make_visualizer(discr, discr.order + if discr.dim == 2 else discr.order) + initname = initializer.__class__.__name__ + eosname = eos.__class__.__name__ + init_message = make_init_message(dim=dim, order=order, + nelements=local_nelements, + global_nelements=global_nelements, + dt=current_dt, t_final=t_final, nstatus=nstatus, + nviz=nviz, cfl=current_cfl, + constant_cfl=constant_cfl, initname=initname, + eosname=eosname, casename=casename) + if rank == 0: + logger.info(init_message) + + get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, + dt=current_dt, cfl=current_cfl, eos=eos, + t_final=t_final, constant_cfl=constant_cfl) + def my_av(state): + return (heat_operator(discr,alpha=1.0e-3,w=state) ) + + def my_rhs(t, state): + return ( + inviscid_operator(discr, q=state, t=t,boundaries=boundaries, eos=eos) + + artificial_viscosity(discr,t=t, r=state, eos=eos, boundaries=boundaries, alpha=4.0e-2) + ) + #return ( + # inviscid_operator(discr, q=state, t=t,boundaries=boundaries, eos=eos) + # + artificial_viscosity(discr, r=state, eos=eos, boundaries=boundaries, alpha=1.0e-3) + + def my_checkpoint(step, t, dt, state): + return sim_checkpoint(discr, visualizer, eos, q=state, + vizname=casename, step=step, + t=t, dt=dt, nstatus=nstatus, nviz=nviz, + exittol=exittol, constant_cfl=constant_cfl, comm=comm) + + try: + (current_step, current_t, current_state) = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + checkpoint=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final) + except ExactSolutionMismatch as ex: + current_step = ex.step + current_t = ex.t + current_state = ex.state + + # if current_t != checkpoint_t: + if rank == 0: + logger.info("Checkpointing final state ...") + my_checkpoint(current_step, t=current_t, + dt=(current_t - checkpoint_t), + state=current_state) + + if current_t - t_final < 0: + raise ValueError("Simulation exited abnormally") + + +if __name__ == "__main__": + logging.basicConfig(format="%(message)s", level=logging.INFO) + main() + +# vim: foldmethod=marker diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index e873ae117..a28ce7187 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -5,6 +5,7 @@ ^^^^^^^^^^^^^^^^^^^^^ .. autoclass:: Vortex2D .. autoclass:: SodShock1D +.. autoclass:: DoubleMachReflection .. autoclass:: Lump .. autoclass:: Uniform """ @@ -212,6 +213,120 @@ def __call__(self, t, x_vec, eos=IdealSingleGas()): return flat_obj_array(mass, energy, mom) +class DoubleMachReflection: + r"""Implement the double shock reflection problem + + - Woodward and Collela + + The inital condition is defined + + .. math:: + + (\rho,u,v,P) = + + This function only serves as an initial condition + + .. automethod:: __init__ + .. automethod:: __call__ + """ + + def __init__( + self,dim=2, x0=1.0/6.0, us=4.0 + ): + """Initialize initial condition options + + Parameters + ---------- + dim: int + dimension of domain, must be 2 + x0: float + location of shock + us: float + shock speed + """ + self._x0 = x0 + self._dim = dim + self._us = us + + def __call__(self, t, x_vec, eos=IdealSingleGas()): + """ + Create the 1D Sod's shock solution at locations *x_vec*. + + Parameters + ---------- + t: float + Current time at which the solution is desired (unused) + x_vec: numpy.ndarray + Nodal coordinates + eos: :class:`mirgecom.eos.GasEOS` + Equation of state class to be used in construction of soln (if needed) + """ + assert self._dim == 2, "only defined for dim=2" + + gm1 = eos.gamma() - 1.0 + gmn1 = 1.0 / gm1 + x_rel = x_vec[0] + y_rel = x_vec[1] + actx = x_rel.array_context + + zeros = 0*x_rel + + x0 = zeros + self._x0 + us = zeros + self._us + t = zeros + t + + + #Mach 10 + #rhol = zeros + 8.0 + #rhor = zeros + 1.4 + + #ul = zeros + 8.25*np.cos(np.pi/6.0) + #ur = zeros + 0.0 + #vl = zeros - 8.25*np.sin(np.pi/6.0) + #vr = zeros + 0.0 + #rhoel = zeros + gmn1 * 116.5 + #rhoer = zeros + gmn1 * 1.0 + + #Mach 2.0 + #rhol = zeros + 2.666666*1.4 + #rhor = zeros + 1.4 + + #ul = zeros + 1.25*np.cos(np.pi/6.0) + #ur = zeros + 0.0 + #vl = zeros - 1.25*np.sin(np.pi/6.0) + #vr = zeros + 0.0 + #rhoel = zeros + gmn1 * 4.5 + #rhoer = zeros + gmn1 * 1.0 + + #Mach 4.0 + rhol = zeros + 4.57142857*1.4 + rhor = zeros + 1.4 + + ul = zeros + 3.125*np.cos(np.pi/6.0) + ur = zeros + 0.0 + vl = zeros - 3.125*np.sin(np.pi/6.0) + vr = zeros + 0.0 + rhoel = zeros + gmn1 * 18.5 + rhoer = zeros + gmn1 * 1.0 + + #yesno = x_rel > (x0 + y_rel/np.sqrt(3.0) + 2.0*us*t/np.sqrt(3.0)) + #mass = actx.np.where(yesno, rhor, rhol) + #rhoe = actx.np.where(yesno, rhoer, rhoel) + #u = actx.np.where(yesno, ur, ul) + #v = actx.np.where(yesno, vr, vl) + xinter = (x0 + y_rel/np.sqrt(3.0) + 2.0*us*t/np.sqrt(3.0)) + sigma=0.05 + xtanh = 1.0/sigma*(x_rel-xinter) + mass = rhol/2.0*(actx.np.tanh(-xtanh)+1.0)+rhor/2.0*(actx.np.tanh(xtanh)+1.0) + rhoe = rhoel/2.0*(actx.np.tanh(-xtanh)+1.0)+rhoer/2.0*(actx.np.tanh(xtanh)+1.0) + u = ul/2.0*(actx.np.tanh(-xtanh)+1.0)+ur/2.0*(actx.np.tanh(xtanh)+1.0) + v = vl/2.0*(actx.np.tanh(-xtanh)+1.0)+vr/2.0*(actx.np.tanh(xtanh)+1.0) + rhou = mass*u + rhov = mass*v + energy = rhoe + 0.5*mass*(u*u + v*v) + + return flat_obj_array(mass, energy, rhou, rhov) + class Lump: r"""Implement an N-dimensional Gaussian lump of mass. From 0f132ebeca5b1fb5f422128cd4bcf0048c22f217 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Fri, 20 Nov 2020 11:54:38 -0800 Subject: [PATCH 008/385] Fixed error in calculating face fluxes across ranks --- mirgecom/artificial_viscosity.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 7711cd7fc..4d9679e25 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -96,8 +96,8 @@ def my_facialflux_r_interior(q): #partition boundaries flux #flux across partition boundaries def my_facialflux_r_partition(q): - qin = cross_rank_trace_pairs(discr,q) - return sum(_facial_flux_r(discr,q_tpair=part_pair) for part_pair in cross_rank_trace_pairs(discr,make_obj_array([q])) ) + qin = cross_rank_trace_pairs(discr,make_obj_array([q])) + return sum(_facial_flux_r(discr,q_tpair=part_pair) for part_pair in qin ) pbf_r = obj_array_vectorize(my_facialflux_r_partition,r) @@ -159,7 +159,7 @@ def my_facialflux_q_interior(q): #flux across partition boundaries def my_facialflux_q_partition(q): qin = cross_rank_trace_pairs(discr,q) - return sum(_facial_flux_q(discr,q_tpair=part_pair) for part_pair in cross_rank_trace_pairs(discr,make_obj_array([q])) ) + return sum(_facial_flux_q(discr,q_tpair=part_pair) for part_pair in qin) pbf_q = obj_array_vectorize(my_facialflux_q_partition,q) #pbf_q = sum( From 40656f36bfc206c2b8795536ce3851ca9b9347b2 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Fri, 20 Nov 2020 12:02:10 -0800 Subject: [PATCH 009/385] Removed unessecary reference to heat operator --- examples/doublemach-mpi.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 6410e0e36..fc3f82fc1 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -133,8 +133,6 @@ def main(ctx_factory=cl.create_some_context): get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, dt=current_dt, cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) - def my_av(state): - return (heat_operator(discr,alpha=1.0e-3,w=state) ) def my_rhs(t, state): return ( From d49de5992eee602fdce2d9722d810c4645ed51ff Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Fri, 20 Nov 2020 12:04:59 -0800 Subject: [PATCH 010/385] Fixed error in not distributing grid --- examples/doublemach-mpi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index fc3f82fc1..d583259a2 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -103,8 +103,10 @@ def main(ctx_factory=cl.create_some_context): from meshmode.mesh.io import read_gmsh, generate_gmsh, ScriptWithFilesSource - local_mesh = read_gmsh("doubleMach1.msh",force_ambient_dim=2) - global_nelements = local_mesh.nelements + + gen_grid = partial(read_gmsh,"doubleMach2.msh",force_ambient_dim=2) + + local_mesh, global_nelements = create_parallel_grid(comm, gen_grid) local_nelements = local_mesh.nelements From 80e1a62f2da68c79404e373fc13d6e0105558c6d Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Fri, 20 Nov 2020 12:37:59 -0800 Subject: [PATCH 011/385] Added shock indicator and integrated with artificial viscosity --- mirgecom/artificial_viscosity.py | 9 ++++++++- mirgecom/tag_cells.py | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 4d9679e25..4459966cb 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -37,6 +37,7 @@ ) from grudge.symbolic.primitives import TracePair from mirgecom.euler import split_conserved, join_conserved +from mirgecom.tag_cells import smoothness_indicator def _facial_flux_r(discr, q_tpair): @@ -73,6 +74,12 @@ def artificial_viscosity(discr, t, eos, boundaries, r, alpha): r"""Compute artifical viscosity for the euler equations """ + #Get smoothness indicator + epsilon = np.zeros((2+discr.dim,),dtype=object) + indicator = make_obj_array([smoothness_indicator(r[0],discr)]) + for i in range(2+discr.dim): + epsilon[i] = indicator + #compute dissapation flux #Cannot call weak_grad on obj of nd arrays, use obj_array_vectorize as work around @@ -139,7 +146,7 @@ def my_facialflux_r_boundary(sol_ext,sol_int): #Compute q, half way done! - q = discr.inverse_mass( -alpha * (dflux_r - discr.face_mass(iff_r + pbf_r + dbf_r))) + q = discr.inverse_mass( -alpha * epsilon * (dflux_r - discr.face_mass(iff_r + pbf_r + dbf_r))) #flux of q diff --git a/mirgecom/tag_cells.py b/mirgecom/tag_cells.py index bf7e196d6..2addfe234 100644 --- a/mirgecom/tag_cells.py +++ b/mirgecom/tag_cells.py @@ -62,7 +62,7 @@ def compute_smoothness_indicator(): 0<=idof (so-kappa) + yesnou = indicator > (so+kappa) + sin_indicator = actx.np.where(yesnol,0.5*(1.0+actx.np.sin(np.pi *(indicator - so)/(2.0*kappa))),0.0*indicator) + indicator = actx.np.where(yesnou,1.0+0.0*indicator,sin_indicator) return indicator From 0c25238c987595af28f6e0452cd29db7c07caba9 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Fri, 20 Nov 2020 12:38:30 -0800 Subject: [PATCH 012/385] Added output field for shock indicator --- mirgecom/simutil.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 2ca30e306..9565be569 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -104,6 +104,9 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, from mirgecom.euler import split_conserved cv = split_conserved(discr.dim, q) dependent_vars = eos.dependent_vars(cv) + + from mirgecom.tag_cells import smoothness_indicator + tagedcells = smoothness_indicator(q[0],discr) rank = 0 if comm is not None: @@ -121,7 +124,8 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, if do_viz: io_fields = [ ("cv", cv), - ("dv", dependent_vars) + ("dv", dependent_vars), + ("tagged", tagedcells) ] if exact_soln is not None: exact_list = [ From 96fc18cf15b52a5b53ee8e1f2ac4412bb1922cfb Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Tue, 1 Dec 2020 15:12:11 -0800 Subject: [PATCH 013/385] Modified stepping parameters for scaling tests --- examples/doublemach-mpi.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index d583259a2..1ea5a2bab 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -67,9 +67,9 @@ def main(ctx_factory=cl.create_some_context): order = 3 # tolerate large errors; case is unstable exittol = 2.0 #.2 - t_final = 1.0 + t_final = 0.0001 current_cfl = 0.1 - current_dt = .00031250 + current_dt = .000001 current_t = 0 eos = IdealSingleGas() initializer = DoubleMachReflection(dim) @@ -84,7 +84,7 @@ def main(ctx_factory=cl.create_some_context): sym.DTAG_BOUNDARY("out"): AdiabaticSlipBoundary()} constant_cfl = False nstatus = 10 - nviz = 25 + nviz = 100 rank = 0 checkpoint_t = current_t current_step = 0 @@ -96,11 +96,6 @@ def main(ctx_factory=cl.create_some_context): comm = MPI.COMM_WORLD rank = comm.Get_rank() - #from meshmode.mesh.generation import generate_regular_rect_mesh - #generate_grid = partial(generate_regular_rect_mesh, a=box_ll, - # b=box_ur, n=nel ) - #local_mesh, global_nelements = create_parallel_grid(comm, generate_grid) - from meshmode.mesh.io import read_gmsh, generate_gmsh, ScriptWithFilesSource @@ -139,7 +134,7 @@ def main(ctx_factory=cl.create_some_context): def my_rhs(t, state): return ( inviscid_operator(discr, q=state, t=t,boundaries=boundaries, eos=eos) - + artificial_viscosity(discr,t=t, r=state, eos=eos, boundaries=boundaries, alpha=4.0e-2) + + artificial_viscosity(discr,t=t, r=state, eos=eos, boundaries=boundaries, alpha=2.0e-2) ) #return ( # inviscid_operator(discr, q=state, t=t,boundaries=boundaries, eos=eos) From 446cacea6e57913892d50bb4cc43d7f9b638d652 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Wed, 20 Jan 2021 15:42:22 -0800 Subject: [PATCH 014/385] Fixed artifical viscosity boundary conditions --- mirgecom/artificial_viscosity.py | 41 +++++++------------------ mirgecom/boundary.py | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 30 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 4459966cb..b0e7b8b42 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -108,21 +108,6 @@ def my_facialflux_r_partition(q): pbf_r = obj_array_vectorize(my_facialflux_r_partition,r) - #pbf_r = sum( - # _facial_flux_r(discr, q_tpair=part_pair) - # for part_pair in cross_rank_trace_pairs(discr, r) - #) - - #domain boundary flux basic boundary implementation - #def my_facialflux2(r): - # dir_r = discr.project("vol", BTAG_ALL,make_obj_array([r])) - # dbf_r = _facial_flux( - # discr, - # q_tpair=TracePair(BTAG_ALL,interior=dir_r,exterior=dir_r) - # ) - # return (dbf_r) - #dbf_r = obj_array_vectorize(my_facialflux2,r) - #True boundary implementation #Okay, not sure about this... @@ -146,6 +131,7 @@ def my_facialflux_r_boundary(sol_ext,sol_int): #Compute q, half way done! + #q = discr.inverse_mass( -alpha * (dflux_r - discr.face_mass(iff_r + pbf_r + dbf_r))) q = discr.inverse_mass( -alpha * epsilon * (dflux_r - discr.face_mass(iff_r + pbf_r + dbf_r))) #flux of q @@ -169,21 +155,16 @@ def my_facialflux_q_partition(q): return sum(_facial_flux_q(discr,q_tpair=part_pair) for part_pair in qin) pbf_q = obj_array_vectorize(my_facialflux_q_partition,q) - #pbf_q = sum( - # _facial_flux_q(discr, q_tpair=part_pair) - # for part_pair in cross_rank_trace_pairs(discr, q) - #) - - def my_facialflux_q_boundary(q): - #dombain boundary flux - dir_q = discr.project("vol",BTAG_ALL, q); - dbf_q = _facial_flux_q( - discr, - q_tpair=TracePair(BTAG_ALL,interior=dir_q,exterior=dir_q) - ) - return(dbf_q) - - dbf_q = obj_array_vectorize(my_facialflux_q_boundary,q) + + dbf_q = 0.0*iff_q + for btag in boundaries: + def my_facialflux_q_boundary(sol_ext,sol_int): + q_tpair = TracePair(btag,interior=sol_int,exterior=sol_ext) + return _facial_flux_q(discr,q_tpair=q_tpair) + q_ext=boundaries[btag].av(discr,eos=eos,btag=btag,t=t,q=q) + q_int=discr.project("vol",btag,q) + dbf_q = dbf_q + obj_array_vectorize_n_args(my_facialflux_q_boundary,q_ext,q_int) + #Return the rhs contribution return ( discr.inverse_mass( -dflux_q + discr.face_mass(iff_q + pbf_q + dbf_q) ) ) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 86fe509d2..40cb11f8b 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -83,6 +83,11 @@ def exterior_sol( ext_soln = self._userfunc(t, nodes) return ext_soln + def av( + self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() + ): + return discr.project("vol",btag,q) + class DummyBoundary: """Use the boundary-adjacent solution as the boundary solution. @@ -108,6 +113,11 @@ def exterior_sol( dir_soln = discr.project("vol", btag, q) return dir_soln + def av( + self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() + ): + return discr.project("vol",btag,q) + class AdiabaticSlipBoundary: """Adiabatic slip boundary for inviscid flows. @@ -223,3 +233,44 @@ def exterior_sol( momentum=bndry_cv.momentum) return bndry_soln + + def av( + self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() + ): + # Grab some boundary-relevant data + dim = discr.dim + cv = split_conserved(dim, q) + actx = cv.mass[0].array_context + + # Grab a unit normal to the boundary + normal = thaw(actx, discr.normal(btag)) + + # Get the interior soln + int_soln = discr.project("vol", btag, q) + bndry_cv = split_conserved(dim, int_soln) + + #flip signs on mass and energy + bndry_cv.mass = -1*bndry_cv.mass + bndry_cv.energy = -1*bndry_cv.energy + + #things are in the wrong order here...flip? + for i in range(dim): + tmp = np.zeros(dim, dtype=object) + for j in range(dim): + tmp[j] = bndry_cv.momentum[j][i] + flip = np.dot(tmp,normal) + norm_flip = normal*make_obj_array([flip]) + tmp = tmp - 2.0*norm_flip + for j in range(dim): + bndry_cv.momentum[j][i] = tmp[j] + + #also need to flip momentum sign + bndry_cv.momentum = -1*bndry_cv.momentum + + #replace it here without single valued check + #No reason these arrays should be messed up... + result = np.zeros(2+dim, dtype=object) + result[0] = bndry_cv.mass + result[1] = bndry_cv.energy + result[2:] = bndry_cv.momentum + return(result) From bf67fc04747bb6a93caf0dbad57bb085f1c27616 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 3 Feb 2021 14:48:44 -0600 Subject: [PATCH 015/385] Bring back functions inadvertently removed in hand merge --- mirgecom/boundary.py | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 64f3c897d..7e5f4b93a 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -105,6 +105,18 @@ def boundary_pair( dir_soln = discr.project("vol", btag, q) return TracePair(btag, interior=dir_soln, exterior=dir_soln) + def exterior_sol( + self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() + ): + """Get the interior and exterior solution on the boundary.""" + dir_soln = discr.project("vol", btag, q) + return dir_soln + + def av( + self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() + ): + return discr.project("vol", btag, q) + class AdiabaticSlipBoundary: r"""Boundary condition implementing inviscid slip boundary. @@ -164,6 +176,53 @@ def boundary_pair( return TracePair(btag, interior=int_soln, exterior=bndry_soln) + def exterior_sol( + self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() + ): + """Get the interior and exterior solution on the boundary. + The exterior solution is set such that there will be vanishing + flux through the boundary, preserving mass, momentum (magnitude) and + energy. + rho_plus = rho_minus + v_plus = v_minus - 2 * (v_minus . n_hat) * n_hat + mom_plus = rho_plus * v_plus + E_plus = E_minus + """ + # Grab some boundary-relevant data + dim = discr.dim + cv = split_conserved(dim, q) + actx = cv.mass.array_context + + # Grab a unit normal to the boundary + normal = thaw(actx, discr.normal(btag)) + normal_mag = actx.np.sqrt(np.dot(normal, normal)) + nhat_mult = 1.0 / normal_mag + nhat = normal * make_obj_array([nhat_mult]) + + # Get the interior/exterior solns + int_soln = discr.project("vol", btag, q) + bndry_cv = split_conserved(dim, int_soln) + # bpressure = eos.pressure(bndry_cv) + + # Subtract out the 2*wall-normal component + # of velocity from the velocity at the wall to + # induce an equal but opposite wall-normal (reflected) wave + # preserving the tangential component + wall_velocity = bndry_cv.momentum / make_obj_array([bndry_cv.mass]) + nvel_comp = np.dot(wall_velocity, nhat) # part of velocity normal to wall + wnorm_vel = nhat * make_obj_array([nvel_comp]) # wall-normal velocity vec + wall_velocity = wall_velocity - 2.0 * wnorm_vel # prescribed ext velocity + + # Re-calculate the boundary solution with the new + # momentum + bndry_cv.momentum = wall_velocity * make_obj_array([bndry_cv.mass]) + # bndry_cv.energy = eos.total_energy(bndry_cv, bpressure) + bndry_soln = join_conserved(dim=dim, mass=bndry_cv.mass, + energy=bndry_cv.energy, + momentum=bndry_cv.momentum) + + return bndry_soln + def av( self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() ): From f52b8b61e5e7adc6d0fb92c41f30db9cfb6d3f2d Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Wed, 3 Feb 2021 16:52:16 -0800 Subject: [PATCH 016/385] Minimum changes to bring boundary conditions up to date with mainline emirge --- mirgecom/boundary.py | 53 ++++++++++++++++++-------------------------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 7e5f4b93a..54153edc0 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -180,6 +180,7 @@ def exterior_sol( self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() ): """Get the interior and exterior solution on the boundary. + The exterior solution is set such that there will be vanishing flux through the boundary, preserving mass, momentum (magnitude) and energy. @@ -194,32 +195,24 @@ def exterior_sol( actx = cv.mass.array_context # Grab a unit normal to the boundary - normal = thaw(actx, discr.normal(btag)) - normal_mag = actx.np.sqrt(np.dot(normal, normal)) - nhat_mult = 1.0 / normal_mag - nhat = normal * make_obj_array([nhat_mult]) + nhat = thaw(actx, discr.normal(btag)) # Get the interior/exterior solns int_soln = discr.project("vol", btag, q) - bndry_cv = split_conserved(dim, int_soln) - # bpressure = eos.pressure(bndry_cv) + int_cv = split_conserved(dim, int_soln) # Subtract out the 2*wall-normal component # of velocity from the velocity at the wall to # induce an equal but opposite wall-normal (reflected) wave # preserving the tangential component - wall_velocity = bndry_cv.momentum / make_obj_array([bndry_cv.mass]) - nvel_comp = np.dot(wall_velocity, nhat) # part of velocity normal to wall - wnorm_vel = nhat * make_obj_array([nvel_comp]) # wall-normal velocity vec - wall_velocity = wall_velocity - 2.0 * wnorm_vel # prescribed ext velocity - - # Re-calculate the boundary solution with the new - # momentum - bndry_cv.momentum = wall_velocity * make_obj_array([bndry_cv.mass]) - # bndry_cv.energy = eos.total_energy(bndry_cv, bpressure) - bndry_soln = join_conserved(dim=dim, mass=bndry_cv.mass, - energy=bndry_cv.energy, - momentum=bndry_cv.momentum) + mom_normcomp = np.dot(int_cv.momentum, nhat) # wall-normal component + wnorm_mom = nhat * mom_normcomp # wall-normal mom vec + ext_mom = int_cv.momentum - 2.0 * wnorm_mom # prescribed ext momentum + + # Form the external boundary solution with the new momentum + bndry_soln = join_conserved(dim=dim, mass=int_cv.mass, + energy=int_cv.energy, + momentum=ext_mom) return bndry_soln @@ -236,30 +229,28 @@ def av( # Get the interior soln int_soln = discr.project("vol", btag, q) - bndry_cv = split_conserved(dim, int_soln) + bndry_q = split_conserved(dim, int_soln) + + #create result array to fill + result = np.zeros(2+dim, dtype=object) # flip signs on mass and energy - bndry_cv.mass = -1*bndry_cv.mass - bndry_cv.energy = -1*bndry_cv.energy + result[0] = -1.0*bndry_q.mass + result[1] = -1.0*bndry_q.energy + + # This needs to be removed + result[2:] = bndry_q.momentum # things are in the wrong order here...flip? for i in range(dim): tmp = np.zeros(dim, dtype=object) for j in range(dim): - tmp[j] = bndry_cv.momentum[j][i] + tmp[j] = bndry_q.momentum[j][i] flip = np.dot(tmp, normal) norm_flip = normal*make_obj_array([flip]) tmp = tmp - 2.0*norm_flip for j in range(dim): - bndry_cv.momentum[j][i] = tmp[j] + result[2+j][i] = -1.0*tmp[j] - # also need to flip momentum sign - bndry_cv.momentum = -1*bndry_cv.momentum - # replace it here without single valued check - # No reason these arrays should be messed up... - result = np.zeros(2+dim, dtype=object) - result[0] = bndry_cv.mass - result[1] = bndry_cv.energy - result[2:] = bndry_cv.momentum return(result) From 703869a268e849dfe4039db106fd3fc59f201f2d Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 3 Feb 2021 20:26:06 -0600 Subject: [PATCH 017/385] Silence flake8. --- examples/doublemach-mpi.py | 147 ++++++++++++++--------- mirgecom/artificial_viscosity.py | 193 ++++++++++++++++--------------- mirgecom/boundary.py | 3 +- mirgecom/initializers.py | 73 ++++++------ mirgecom/simutil.py | 4 +- mirgecom/tag_cells.py | 76 +++++++----- 6 files changed, 275 insertions(+), 221 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 1ea5a2bab..3b4ebd8de 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -47,11 +47,9 @@ from mirgecom.integrators import rk4_step from mirgecom.steppers import advance_state from mirgecom.boundary import AdiabaticSlipBoundary, PrescribedBoundary -from mirgecom.initializers import DoubleMachReflection, SodShock1D +from mirgecom.initializers import DoubleMachReflection from mirgecom.eos import IdealSingleGas -from pytools.obj_array import obj_array_vectorize - logger = logging.getLogger(__name__) @@ -59,29 +57,33 @@ def main(ctx_factory=cl.create_some_context): cl_ctx = ctx_factory() queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + actx = PyOpenCLArrayContext( + queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)) + ) dim = 2 - nel = (40,10) + # nel = (40, 10) order = 3 # tolerate large errors; case is unstable - exittol = 2.0 #.2 + exittol = 2.0 # .2 t_final = 0.0001 current_cfl = 0.1 - current_dt = .000001 + current_dt = 0.000001 current_t = 0 eos = IdealSingleGas() initializer = DoubleMachReflection(dim) - #initializer = SodShock1D(dim,x0=0.5) + # initializer = SodShock1D(dim,x0=0.5) casename = "sod1d" - #boundaries = {BTAG_ALL: AdiabaticSlipBoundary()} + # boundaries = {BTAG_ALL: AdiabaticSlipBoundary()} from grudge import sym - boundaries = {sym.DTAG_BOUNDARY("ic1"): PrescribedBoundary(initializer), - sym.DTAG_BOUNDARY("ic2"): PrescribedBoundary(initializer), - sym.DTAG_BOUNDARY("ic3"): PrescribedBoundary(initializer), - sym.DTAG_BOUNDARY("wall"): AdiabaticSlipBoundary(), - sym.DTAG_BOUNDARY("out"): AdiabaticSlipBoundary()} + + boundaries = { + sym.DTAG_BOUNDARY("ic1"): PrescribedBoundary(initializer), + sym.DTAG_BOUNDARY("ic2"): PrescribedBoundary(initializer), + sym.DTAG_BOUNDARY("ic3"): PrescribedBoundary(initializer), + sym.DTAG_BOUNDARY("wall"): AdiabaticSlipBoundary(), + sym.DTAG_BOUNDARY("out"): AdiabaticSlipBoundary(), + } constant_cfl = False nstatus = 10 nviz = 100 @@ -89,69 +91,99 @@ def main(ctx_factory=cl.create_some_context): checkpoint_t = current_t current_step = 0 timestepper = rk4_step - box_ll = (0.0,0.0) - box_ur = (4.0,1.0) + # box_ll = (0.0, 0.0) + # box_ur = (4.0, 1.0) from mpi4py import MPI + comm = MPI.COMM_WORLD rank = comm.Get_rank() + from meshmode.mesh.io import read_gmsh + gen_grid = partial(read_gmsh, "doubleMach2.msh", force_ambient_dim=2) - from meshmode.mesh.io import read_gmsh, generate_gmsh, ScriptWithFilesSource - - gen_grid = partial(read_gmsh,"doubleMach2.msh",force_ambient_dim=2) - local_mesh, global_nelements = create_parallel_grid(comm, gen_grid) local_nelements = local_mesh.nelements - discr = EagerDGDiscretization( - actx, local_mesh, order=order, mpi_communicator=comm - ) + discr = EagerDGDiscretization(actx, local_mesh, order=order, + mpi_communicator=comm) nodes = thaw(actx, discr.nodes()) current_state = initializer(0, nodes) - #visualizer = make_visualizer(discr, discr.order + 3 + # visualizer = make_visualizer(discr, discr.order + 3 # if discr.dim == 2 else discr.order) - visualizer = make_visualizer(discr, discr.order - if discr.dim == 2 else discr.order) + visualizer = make_visualizer(discr, + discr.order if discr.dim == 2 else discr.order) initname = initializer.__class__.__name__ eosname = eos.__class__.__name__ - init_message = make_init_message(dim=dim, order=order, - nelements=local_nelements, - global_nelements=global_nelements, - dt=current_dt, t_final=t_final, nstatus=nstatus, - nviz=nviz, cfl=current_cfl, - constant_cfl=constant_cfl, initname=initname, - eosname=eosname, casename=casename) + init_message = make_init_message( + dim=dim, + order=order, + nelements=local_nelements, + global_nelements=global_nelements, + dt=current_dt, + t_final=t_final, + nstatus=nstatus, + nviz=nviz, + cfl=current_cfl, + constant_cfl=constant_cfl, + initname=initname, + eosname=eosname, + casename=casename, + ) if rank == 0: logger.info(init_message) - get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, - dt=current_dt, cfl=current_cfl, eos=eos, - t_final=t_final, constant_cfl=constant_cfl) + get_timestep = partial( + inviscid_sim_timestep, + discr=discr, + t=current_t, + dt=current_dt, + cfl=current_cfl, + eos=eos, + t_final=t_final, + constant_cfl=constant_cfl, + ) def my_rhs(t, state): - return ( - inviscid_operator(discr, q=state, t=t,boundaries=boundaries, eos=eos) - + artificial_viscosity(discr,t=t, r=state, eos=eos, boundaries=boundaries, alpha=2.0e-2) - ) - #return ( - # inviscid_operator(discr, q=state, t=t,boundaries=boundaries, eos=eos) - # + artificial_viscosity(discr, r=state, eos=eos, boundaries=boundaries, alpha=1.0e-3) + return inviscid_operator( + discr, q=state, t=t, boundaries=boundaries, eos=eos + ) + artificial_viscosity( + discr, t=t, r=state, eos=eos, boundaries=boundaries, alpha=2.0e-2 + ) + # return ( + # inviscid_operator(discr, q=state, t=t,boundaries=boundaries, eos=eos) + # + artificial_viscosity(discr, r=state, eos=eos, boundaries=boundaries, + # alpha=1.0e-3) def my_checkpoint(step, t, dt, state): - return sim_checkpoint(discr, visualizer, eos, q=state, - vizname=casename, step=step, - t=t, dt=dt, nstatus=nstatus, nviz=nviz, - exittol=exittol, constant_cfl=constant_cfl, comm=comm) + return sim_checkpoint( + discr, + visualizer, + eos, + q=state, + vizname=casename, + step=step, + t=t, + dt=dt, + nstatus=nstatus, + nviz=nviz, + exittol=exittol, + constant_cfl=constant_cfl, + comm=comm, + ) try: - (current_step, current_t, current_state) = \ - advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=my_checkpoint, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) + (current_step, current_t, current_state) = advance_state( + rhs=my_rhs, + timestepper=timestepper, + checkpoint=my_checkpoint, + get_timestep=get_timestep, + state=current_state, + t=current_t, + t_final=t_final, + ) except ExactSolutionMismatch as ex: current_step = ex.step current_t = ex.t @@ -160,9 +192,12 @@ def my_checkpoint(step, t, dt, state): # if current_t != checkpoint_t: if rank == 0: logger.info("Checkpointing final state ...") - my_checkpoint(current_step, t=current_t, - dt=(current_t - checkpoint_t), - state=current_state) + my_checkpoint( + current_step, + t=current_t, + dt=(current_t - checkpoint_t), + state=current_state, + ) if current_t - t_final < 0: raise ValueError("Simulation exited abnormally") diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index b0e7b8b42..3204bb854 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -1,4 +1,4 @@ -r""":mod:`mirgecom.artificial viscosity` applys and artifical viscosity to the euler equations +r""":mod:`mirgecom.artificial viscosity` Artificial viscocity for Euler. """ __copyright__ = """ @@ -25,24 +25,24 @@ THE SOFTWARE. """ -#from dataclasses import dataclass +# from dataclasses import dataclass import numpy as np -from pytools.obj_array import make_obj_array, obj_array_vectorize, flat_obj_array,obj_array_vectorize_n_args +from pytools.obj_array import ( + make_obj_array, + obj_array_vectorize, + obj_array_vectorize_n_args, +) from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa -from grudge.eager import ( - interior_trace_pair, - cross_rank_trace_pairs -) +from grudge.eager import interior_trace_pair, cross_rank_trace_pairs from grudge.symbolic.primitives import TracePair -from mirgecom.euler import split_conserved, join_conserved +# from mirgecom.euler import split_conserved, join_conserved from mirgecom.tag_cells import smoothness_indicator def _facial_flux_r(discr, q_tpair): - dim = discr.dim actx = q_tpair[0].int.array_context flux_dis = q_tpair.avg @@ -50,122 +50,129 @@ def _facial_flux_r(discr, q_tpair): normal = thaw(actx, discr.normal(q_tpair.dd)) flux_out = flux_dis * normal - + # Can't do it here... "obj arrays not allowed on compute device" - #def flux_calc(flux): + # def flux_calc(flux): # return (flux * normal) - #flux_out = obj_array_vectorize(flux_calc,flux_dis) + # flux_out = obj_array_vectorize(flux_calc,flux_dis) return discr.project(q_tpair.dd, "all_faces", flux_out) -def _facial_flux_q(discr, q_tpair): - dim = discr.dim +def _facial_flux_q(discr, q_tpair): actx = q_tpair[0].int.array_context normal = thaw(actx, discr.normal(q_tpair.dd)) - flux_out = np.dot(q_tpair.avg,normal) + flux_out = np.dot(q_tpair.avg, normal) return discr.project(q_tpair.dd, "all_faces", flux_out) -def artificial_viscosity(discr, t, eos, boundaries, r, alpha): - r"""Compute artifical viscosity for the euler equations - """ - #Get smoothness indicator - epsilon = np.zeros((2+discr.dim,),dtype=object) - indicator = make_obj_array([smoothness_indicator(r[0],discr)]) - for i in range(2+discr.dim): +def artificial_viscosity(discr, t, eos, boundaries, r, alpha): + r"""Compute artifical viscosity for the euler equations""" + # Get smoothness indicator + epsilon = np.zeros((2 + discr.dim,), dtype=object) + indicator = make_obj_array([smoothness_indicator(r[0], discr)]) + for i in range(2 + discr.dim): epsilon[i] = indicator - #compute dissapation flux + # compute dissapation flux + + # Cannot call weak_grad on obj of nd arrays + # use obj_array_vectorize as work around + dflux_r = obj_array_vectorize(discr.weak_grad, r) - #Cannot call weak_grad on obj of nd arrays, use obj_array_vectorize as work around - dflux_r = obj_array_vectorize(discr.weak_grad,r) - - #interior face flux + # interior face flux - #Doesn't work: something related to obj on compute device - #Not 100% on reason - #qin = interior_trace_pair(discr,r) - #iff_r = _facial_flux(discr,q_tpair=qin) + # Doesn't work: something related to obj on compute device + # Not 100% on reason + # qin = interior_trace_pair(discr,r) + # iff_r = _facial_flux(discr,q_tpair=qin) - #Work around? + # Work around? def my_facialflux_r_interior(q): - qin = interior_trace_pair(discr,make_obj_array([q])) - return _facial_flux_r(discr,q_tpair=qin) + qin = interior_trace_pair(discr, make_obj_array([q])) + return _facial_flux_r(discr, q_tpair=qin) - iff_r = obj_array_vectorize(my_facialflux_r_interior,r) - + iff_r = obj_array_vectorize(my_facialflux_r_interior, r) - #partition boundaries flux - #flux across partition boundaries + # partition boundaries flux + # flux across partition boundaries def my_facialflux_r_partition(q): - qin = cross_rank_trace_pairs(discr,make_obj_array([q])) - return sum(_facial_flux_r(discr,q_tpair=part_pair) for part_pair in qin ) - - pbf_r = obj_array_vectorize(my_facialflux_r_partition,r) - - - #True boundary implementation - #Okay, not sure about this... - #What I am attempting: + qin = cross_rank_trace_pairs(discr, make_obj_array([q])) + return sum(_facial_flux_r(discr, q_tpair=part_pair) for part_pair in qin) + + pbf_r = obj_array_vectorize(my_facialflux_r_partition, r) + + # True boundary implementation + # Okay, not sure about this... + # What I am attempting: # 1. Loop through all the boundaries - # 2. Define a function my_TP that performes the trace pair for the given boundary - # given a solution variable + # 2. Define a function my_TP that performes the trace pair for the given + # boundary given a solution variable # 3. Get the external solution from the boundary routine # 4. Get hte projected internal solution - # 5. Compute the boundary flux as a sum over boundaries, using the obj_array_vectorize to - # pass each solution variable one at a time + # 5. Compute the boundary flux as a sum over boundaries, using the + # obj_array_vectorize to pass each solution variable one at a time # DO I really need to do this like this? - dbf_r = 0.0*iff_r + dbf_r = 0.0 * iff_r for btag in boundaries: - def my_facialflux_r_boundary(sol_ext,sol_int): - q_tpair = TracePair(btag,interior=make_obj_array([sol_int]),exterior=make_obj_array([sol_ext])) - return _facial_flux_r(discr,q_tpair=q_tpair) - r_ext=boundaries[btag].exterior_sol(discr,eos=eos,btag=btag,t=t,q=r) - r_int=discr.project("vol",btag,r) - dbf_r = dbf_r + obj_array_vectorize_n_args(my_facialflux_r_boundary,r_ext,r_int) - - - #Compute q, half way done! - #q = discr.inverse_mass( -alpha * (dflux_r - discr.face_mass(iff_r + pbf_r + dbf_r))) - q = discr.inverse_mass( -alpha * epsilon * (dflux_r - discr.face_mass(iff_r + pbf_r + dbf_r))) - - #flux of q - - #Again we need to vectorize - #q is a object array of object arrays (dim,) of DOFArrays (?) - dflux_q = obj_array_vectorize(discr.weak_div,q) - - #interior face flux of q + + def my_facialflux_r_boundary(sol_ext, sol_int): + q_tpair = TracePair( + btag, + interior=make_obj_array([sol_int]), + exterior=make_obj_array([sol_ext]), + ) + return _facial_flux_r(discr, q_tpair=q_tpair) + + r_ext = boundaries[btag].exterior_sol(discr, eos=eos, btag=btag, t=t, q=r) + r_int = discr.project("vol", btag, r) + dbf_r = dbf_r + obj_array_vectorize_n_args( + my_facialflux_r_boundary, r_ext, r_int + ) + + # Compute q, half way done! + # q = discr.inverse_mass(-alpha*(dflux_r-discr.face_mass(iff_r + pbf_r + dbf_r))) + q = discr.inverse_mass( + -alpha * epsilon * (dflux_r - discr.face_mass(iff_r + pbf_r + dbf_r)) + ) + + # flux of q + + # Again we need to vectorize + # q is a object array of object arrays (dim,) of DOFArrays (?) + dflux_q = obj_array_vectorize(discr.weak_div, q) + + # interior face flux of q def my_facialflux_q_interior(q): - qin = interior_trace_pair(discr,q) - iff_q = _facial_flux_q(discr, q_tpair=qin) - return (iff_q) + qin = interior_trace_pair(discr, q) + iff_q = _facial_flux_q(discr, q_tpair=qin) + return iff_q - iff_q = obj_array_vectorize(my_facialflux_q_interior,q) - + iff_q = obj_array_vectorize(my_facialflux_q_interior, q) - #flux across partition boundaries + # flux across partition boundaries def my_facialflux_q_partition(q): - qin = cross_rank_trace_pairs(discr,q) - return sum(_facial_flux_q(discr,q_tpair=part_pair) for part_pair in qin) - - pbf_q = obj_array_vectorize(my_facialflux_q_partition,q) + qin = cross_rank_trace_pairs(discr, q) + return sum(_facial_flux_q(discr, q_tpair=part_pair) for part_pair in qin) + + pbf_q = obj_array_vectorize(my_facialflux_q_partition, q) - dbf_q = 0.0*iff_q + dbf_q = 0.0 * iff_q for btag in boundaries: - def my_facialflux_q_boundary(sol_ext,sol_int): - q_tpair = TracePair(btag,interior=sol_int,exterior=sol_ext) - return _facial_flux_q(discr,q_tpair=q_tpair) - q_ext=boundaries[btag].av(discr,eos=eos,btag=btag,t=t,q=q) - q_int=discr.project("vol",btag,q) - dbf_q = dbf_q + obj_array_vectorize_n_args(my_facialflux_q_boundary,q_ext,q_int) - - - #Return the rhs contribution - return ( discr.inverse_mass( -dflux_q + discr.face_mass(iff_q + pbf_q + dbf_q) ) ) - + + def my_facialflux_q_boundary(sol_ext, sol_int): + q_tpair = TracePair(btag, interior=sol_int, exterior=sol_ext) + return _facial_flux_q(discr, q_tpair=q_tpair) + + q_ext = boundaries[btag].av(discr, eos=eos, btag=btag, t=t, q=q) + q_int = discr.project("vol", btag, q) + dbf_q = dbf_q + obj_array_vectorize_n_args( + my_facialflux_q_boundary, q_ext, q_int + ) + + # Return the rhs contribution + return discr.inverse_mass(-dflux_q + discr.face_mass(iff_q + pbf_q + dbf_q)) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 54153edc0..e8162a6c0 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -231,7 +231,7 @@ def av( int_soln = discr.project("vol", btag, q) bndry_q = split_conserved(dim, int_soln) - #create result array to fill + # create result array to fill result = np.zeros(2+dim, dtype=object) # flip signs on mass and energy @@ -252,5 +252,4 @@ def av( for j in range(dim): result[2+j][i] = -1.0*tmp[j] - return(result) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 251cfcec4..51db02520 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -305,6 +305,7 @@ def __call__(self, t, x_vec, eos=IdealSingleGas()): return flat_obj_array(mass, energy, mom) + class DoubleMachReflection: r"""Implement the double shock reflection problem @@ -314,7 +315,7 @@ class DoubleMachReflection: .. math:: - (\rho,u,v,P) = + (\rho,u,v,P) = This function only serves as an initial condition @@ -323,7 +324,7 @@ class DoubleMachReflection: """ def __init__( - self,dim=2, x0=1.0/6.0, us=4.0 + self, dim=2, x0=1.0/6.0, us=4.0 ): """Initialize initial condition options @@ -337,7 +338,7 @@ def __init__( shock speed """ self._x0 = x0 - self._dim = dim + self._dim = dim self._us = us def __call__(self, t, x_vec, eos=IdealSingleGas()): @@ -366,34 +367,33 @@ def __call__(self, t, x_vec, eos=IdealSingleGas()): x0 = zeros + self._x0 us = zeros + self._us t = zeros + t - - - #Mach 10 - #rhol = zeros + 8.0 - #rhor = zeros + 1.4 - - #ul = zeros + 8.25*np.cos(np.pi/6.0) - #ur = zeros + 0.0 - #vl = zeros - 8.25*np.sin(np.pi/6.0) - #vr = zeros + 0.0 - #rhoel = zeros + gmn1 * 116.5 - #rhoer = zeros + gmn1 * 1.0 - - #Mach 2.0 - #rhol = zeros + 2.666666*1.4 - #rhor = zeros + 1.4 - - #ul = zeros + 1.25*np.cos(np.pi/6.0) - #ur = zeros + 0.0 - #vl = zeros - 1.25*np.sin(np.pi/6.0) - #vr = zeros + 0.0 - #rhoel = zeros + gmn1 * 4.5 - #rhoer = zeros + gmn1 * 1.0 - - #Mach 4.0 + + # Mach 10 + # rhol = zeros + 8.0 + # rhor = zeros + 1.4 + + # ul = zeros + 8.25*np.cos(np.pi/6.0) + # ur = zeros + 0.0 + # vl = zeros - 8.25*np.sin(np.pi/6.0) + # vr = zeros + 0.0 + # rhoel = zeros + gmn1 * 116.5 + # rhoer = zeros + gmn1 * 1.0 + + # Mach 2.0 + # rhol = zeros + 2.666666*1.4 + # rhor = zeros + 1.4 + + # ul = zeros + 1.25*np.cos(np.pi/6.0) + # ur = zeros + 0.0 + # vl = zeros - 1.25*np.sin(np.pi/6.0) + # vr = zeros + 0.0 + # rhoel = zeros + gmn1 * 4.5 + # rhoer = zeros + gmn1 * 1.0 + + # Mach 4.0 rhol = zeros + 4.57142857*1.4 rhor = zeros + 1.4 - + ul = zeros + 3.125*np.cos(np.pi/6.0) ur = zeros + 0.0 vl = zeros - 3.125*np.sin(np.pi/6.0) @@ -401,16 +401,17 @@ def __call__(self, t, x_vec, eos=IdealSingleGas()): rhoel = zeros + gmn1 * 18.5 rhoer = zeros + gmn1 * 1.0 - #yesno = x_rel > (x0 + y_rel/np.sqrt(3.0) + 2.0*us*t/np.sqrt(3.0)) - #mass = actx.np.where(yesno, rhor, rhol) - #rhoe = actx.np.where(yesno, rhoer, rhoel) - #u = actx.np.where(yesno, ur, ul) - #v = actx.np.where(yesno, vr, vl) + # yesno = x_rel > (x0 + y_rel/np.sqrt(3.0) + 2.0*us*t/np.sqrt(3.0)) + # mass = actx.np.where(yesno, rhor, rhol) + # rhoe = actx.np.where(yesno, rhoer, rhoel) + # u = actx.np.where(yesno, ur, ul) + # v = actx.np.where(yesno, vr, vl) xinter = (x0 + y_rel/np.sqrt(3.0) + 2.0*us*t/np.sqrt(3.0)) - sigma=0.05 + sigma = 0.05 xtanh = 1.0/sigma*(x_rel-xinter) mass = rhol/2.0*(actx.np.tanh(-xtanh)+1.0)+rhor/2.0*(actx.np.tanh(xtanh)+1.0) - rhoe = rhoel/2.0*(actx.np.tanh(-xtanh)+1.0)+rhoer/2.0*(actx.np.tanh(xtanh)+1.0) + rhoe = (rhoel/2.0*(actx.np.tanh(-xtanh)+1.0) + + rhoer/2.0*(actx.np.tanh(xtanh)+1.0)) u = ul/2.0*(actx.np.tanh(-xtanh)+1.0)+ur/2.0*(actx.np.tanh(xtanh)+1.0) v = vl/2.0*(actx.np.tanh(-xtanh)+1.0)+vr/2.0*(actx.np.tanh(xtanh)+1.0) rhou = mass*u diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 9565be569..6b7cbbfd1 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -104,9 +104,9 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, from mirgecom.euler import split_conserved cv = split_conserved(discr.dim, q) dependent_vars = eos.dependent_vars(cv) - + from mirgecom.tag_cells import smoothness_indicator - tagedcells = smoothness_indicator(q[0],discr) + tagedcells = smoothness_indicator(q[0], discr) rank = 0 if comm is not None: diff --git a/mirgecom/tag_cells.py b/mirgecom/tag_cells.py index 2addfe234..5616c81d2 100644 --- a/mirgecom/tag_cells.py +++ b/mirgecom/tag_cells.py @@ -4,7 +4,8 @@ .. math:: - S_e = \frac{\langle u_{N_p} - u_{N_{p-1}}, u_{N_p} - u_{N_{p-1}}\rangle_e}{\langle u_{N_p}, u_{N_p} \rangle_e} + S_e = \frac{\langle u_{N_p} - u_{N_{p-1}}, u_{N_p} - + u_{N_{p-1}}\rangle_e}{\langle u_{N_p}, u_{N_p} \rangle_e} """ __copyright__ = """ @@ -33,43 +34,48 @@ import numpy as np import loopy as lp -from grudge import sym from meshmode.dof_array import DOFArray from modepy import vandermonde -from pytools import memoize_in -from pytools.obj_array import make_obj_array def linear_operator_kernel(): """Apply linear operator to all elements.""" from meshmode.array_context import make_loopy_program + knl = make_loopy_program( """{[iel,idof,j]: 0<=iel (so-kappa) - yesnou = indicator > (so+kappa) - sin_indicator = actx.np.where(yesnol,0.5*(1.0+actx.np.sin(np.pi *(indicator - so)/(2.0*kappa))),0.0*indicator) - indicator = actx.np.where(yesnou,1.0+0.0*indicator,sin_indicator) + # Compute artificail viscosity percentage based on idicator and set parameters + yesnol = indicator > (so - kappa) + yesnou = indicator > (so + kappa) + sin_indicator = actx.np.where( + yesnol, + 0.5 * (1.0 + actx.np.sin(np.pi * (indicator - so) / (2.0 * kappa))), + 0.0 * indicator, + ) + indicator = actx.np.where(yesnou, 1.0 + 0.0 * indicator, sin_indicator) return indicator - From f20059f873a74db5a4c0b73e2f2104f88400fc05 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 3 Feb 2021 20:43:15 -0600 Subject: [PATCH 018/385] Silence pydocstyle --- examples/doublemach-mpi.py | 3 +++ examples/heat-source-mpi.py | 3 +++ mirgecom/artificial_viscosity.py | 5 ++--- mirgecom/boundary.py | 3 +++ mirgecom/initializers.py | 4 ++-- mirgecom/profiling.py | 1 + mirgecom/tag_cells.py | 4 ++-- 7 files changed, 16 insertions(+), 7 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 3b4ebd8de..d23adb80f 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -1,3 +1,5 @@ +"""Demonstrate doublemach reflection.""" + __copyright__ = """ Copyright (C) 2020 University of Illinois Board of Trustees """ @@ -55,6 +57,7 @@ @mpi_entry_point def main(ctx_factory=cl.create_some_context): + """Drive the example.""" cl_ctx = ctx_factory() queue = cl.CommandQueue(cl_ctx) actx = PyOpenCLArrayContext( diff --git a/examples/heat-source-mpi.py b/examples/heat-source-mpi.py index 35e79c8fb..8150305fc 100644 --- a/examples/heat-source-mpi.py +++ b/examples/heat-source-mpi.py @@ -1,3 +1,5 @@ +"""Demonstrate heat source.""" + __copyright__ = "Copyright (C) 2020 University of Illinois Board of Trustees" __license__ = """ @@ -45,6 +47,7 @@ @mpi_entry_point def main(): + """Drive the example.""" cl_ctx = cl.create_some_context() queue = cl.CommandQueue(cl_ctx) actx = PyOpenCLArrayContext(queue, diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 3204bb854..b1c13d723 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -1,5 +1,4 @@ -r""":mod:`mirgecom.artificial viscosity` Artificial viscocity for Euler. -""" +""":mod:`mirgecom.artificial viscosity` Artificial viscocity for Euler.""" __copyright__ = """ Copyright (C) 2020 University of Illinois Board of Trustees @@ -71,7 +70,7 @@ def _facial_flux_q(discr, q_tpair): def artificial_viscosity(discr, t, eos, boundaries, r, alpha): - r"""Compute artifical viscosity for the euler equations""" + r"""Compute artifical viscosity for the euler equations.""" # Get smoothness indicator epsilon = np.zeros((2 + discr.dim,), dtype=object) indicator = make_obj_array([smoothness_indicator(r[0], discr)]) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index e8162a6c0..3b9354566 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -85,6 +85,7 @@ def exterior_sol( def av( self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() ): + """Do artificial viscosity function.""" return discr.project("vol", btag, q) @@ -115,6 +116,7 @@ def exterior_sol( def av( self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() ): + """Do artificial viscosity function.""" return discr.project("vol", btag, q) @@ -219,6 +221,7 @@ def exterior_sol( def av( self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() ): + """Do artificial viscosity function.""" # Grab some boundary-relevant data dim = discr.dim cv = split_conserved(dim, q) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 51db02520..6d784fb83 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -307,7 +307,7 @@ def __call__(self, t, x_vec, eos=IdealSingleGas()): class DoubleMachReflection: - r"""Implement the double shock reflection problem + r"""Implement the double shock reflection problem. - Woodward and Collela @@ -326,7 +326,7 @@ class DoubleMachReflection: def __init__( self, dim=2, x0=1.0/6.0, us=4.0 ): - """Initialize initial condition options + """Initialize initial condition options. Parameters ---------- diff --git a/mirgecom/profiling.py b/mirgecom/profiling.py index a880f98d5..d600d6ce7 100644 --- a/mirgecom/profiling.py +++ b/mirgecom/profiling.py @@ -106,6 +106,7 @@ class PyOpenCLProfilingArrayContext(PyOpenCLArrayContext): """ def __init__(self, queue, allocator=None) -> None: + """Initialize the object.""" super().__init__(queue, allocator) if not queue.properties & cl.command_queue_properties.PROFILING_ENABLE: diff --git a/mirgecom/tag_cells.py b/mirgecom/tag_cells.py index 5616c81d2..3288ae921 100644 --- a/mirgecom/tag_cells.py +++ b/mirgecom/tag_cells.py @@ -1,4 +1,4 @@ -r""":mod:`mirgecom.tag_cells` Computes smoothness indicator +r""":mod:`mirgecom.tag_cells` Compute smoothness indicator. Perssons smoothness indicator: @@ -74,7 +74,7 @@ def compute_smoothness_indicator(): def smoothness_indicator(u, discr): - + """Calculate the smoothness indicator.""" assert isinstance(u, DOFArray) # #@memoize_in(u.array_context, (smoothness_indicator, "get_kernel")) From 1340ea6c2ea74afb163266612755486aa0a3d37f Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Thu, 4 Feb 2021 12:09:17 -0800 Subject: [PATCH 019/385] Remove un-needed make_obj_array calls --- mirgecom/artificial_viscosity.py | 13 ++++++------- mirgecom/boundary.py | 3 +-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index b1c13d723..7ba1d0cb9 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -28,7 +28,6 @@ import numpy as np from pytools.obj_array import ( - make_obj_array, obj_array_vectorize, obj_array_vectorize_n_args, ) @@ -42,7 +41,7 @@ def _facial_flux_r(discr, q_tpair): - actx = q_tpair[0].int.array_context + actx = q_tpair.int.array_context flux_dis = q_tpair.avg @@ -73,7 +72,7 @@ def artificial_viscosity(discr, t, eos, boundaries, r, alpha): r"""Compute artifical viscosity for the euler equations.""" # Get smoothness indicator epsilon = np.zeros((2 + discr.dim,), dtype=object) - indicator = make_obj_array([smoothness_indicator(r[0], discr)]) + indicator = smoothness_indicator(r[0], discr) for i in range(2 + discr.dim): epsilon[i] = indicator @@ -92,7 +91,7 @@ def artificial_viscosity(discr, t, eos, boundaries, r, alpha): # Work around? def my_facialflux_r_interior(q): - qin = interior_trace_pair(discr, make_obj_array([q])) + qin = interior_trace_pair(discr, q) return _facial_flux_r(discr, q_tpair=qin) iff_r = obj_array_vectorize(my_facialflux_r_interior, r) @@ -100,7 +99,7 @@ def my_facialflux_r_interior(q): # partition boundaries flux # flux across partition boundaries def my_facialflux_r_partition(q): - qin = cross_rank_trace_pairs(discr, make_obj_array([q])) + qin = cross_rank_trace_pairs(discr, q) return sum(_facial_flux_r(discr, q_tpair=part_pair) for part_pair in qin) pbf_r = obj_array_vectorize(my_facialflux_r_partition, r) @@ -122,8 +121,8 @@ def my_facialflux_r_partition(q): def my_facialflux_r_boundary(sol_ext, sol_int): q_tpair = TracePair( btag, - interior=make_obj_array([sol_int]), - exterior=make_obj_array([sol_ext]), + interior=sol_int, + exterior=sol_ext, ) return _facial_flux_r(discr, q_tpair=q_tpair) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 4469a3f0b..324bf2689 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -33,7 +33,6 @@ """ import numpy as np -from pytools.obj_array import make_obj_array from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from mirgecom.eos import IdealSingleGas @@ -243,7 +242,7 @@ def av( for j in range(dim): tmp[j] = bndry_q.momentum[j][i] flip = np.dot(tmp, normal) - norm_flip = normal*make_obj_array([flip]) + norm_flip = normal*flip tmp = tmp - 2.0*norm_flip for j in range(dim): result[2+j][i] = -1.0*tmp[j] From 9d32a7a9fef80637a12dd555492f9c08621266e1 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Thu, 4 Feb 2021 13:17:14 -0800 Subject: [PATCH 020/385] Rework boundary conditions to avoid duplicating code --- mirgecom/artificial_viscosity.py | 2 +- mirgecom/boundary.py | 82 +++++++------------------------- 2 files changed, 19 insertions(+), 65 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 7ba1d0cb9..c9add83d2 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -126,7 +126,7 @@ def my_facialflux_r_boundary(sol_ext, sol_int): ) return _facial_flux_r(discr, q_tpair=q_tpair) - r_ext = boundaries[btag].exterior_sol(discr, eos=eos, btag=btag, t=t, q=r) + r_ext = boundaries[btag].exterior_soln(discr, eos=eos, btag=btag, t=t, q=r) r_int = discr.project("vol", btag, r) dbf_r = dbf_r + obj_array_vectorize_n_args( my_facialflux_r_boundary, r_ext, r_int diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 324bf2689..bf5272fc6 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -62,29 +62,21 @@ def __init__(self, userfunc): def boundary_pair(self, discr, q, btag, **kwargs): """Get the interior and exterior solution on the boundary.""" - actx = q[0].array_context - - boundary_discr = discr.discr_from_dd(btag) - nodes = thaw(actx, boundary_discr.nodes()) - ext_soln = self._userfunc(nodes, **kwargs) + ext_soln = self.exterior_soln(self, discr, q, btag, **kwargs) int_soln = discr.project("vol", btag, q) return TracePair(btag, interior=int_soln, exterior=ext_soln) - def exterior_sol( - self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() - ): - """Get the interior solution on the boundary.""" + def exterior_soln(self, discr, q, btag, **kwargs): + """Get the exterior solution on the boundary.""" actx = q[0].array_context boundary_discr = discr.discr_from_dd(btag) nodes = thaw(actx, boundary_discr.nodes()) - ext_soln = self._userfunc(t, nodes) + ext_soln = self._userfunc(nodes, **kwargs) return ext_soln - def av( - self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() - ): - """Do artificial viscosity function.""" + def av(self, discr, q, btag, **kwargs): + """Get the exterior solution on the boundary.""" return discr.project("vol", btag, q) @@ -96,20 +88,16 @@ class DummyBoundary: def boundary_pair(self, discr, q, btag, **kwargs): """Get the interior and exterior solution on the boundary.""" - dir_soln = discr.project("vol", btag, q) + dir_soln = self.exterior_soln(self, discr, q, btag, **kwargs) return TracePair(btag, interior=dir_soln, exterior=dir_soln) - def exterior_sol( - self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() - ): - """Get the interior and exterior solution on the boundary.""" + def exterior_soln(self, discr, q, btag, **kwargs): + """Get the exterior solution on the boundary.""" dir_soln = discr.project("vol", btag, q) return dir_soln - def av( - self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() - ): - """Do artificial viscosity function.""" + def av(self, discr, q, btag, **kwargs): + """Get the exterior solution on the boundary.""" return discr.project("vol", btag, q) @@ -132,48 +120,15 @@ class AdiabaticSlipBoundary: """ def boundary_pair(self, discr, q, btag, **kwargs): - """Get the interior and exterior solution on the boundary. - - The exterior solution is set such that there will be vanishing - flux through the boundary, preserving mass, momentum (magnitude) and - energy. - rho_plus = rho_minus - v_plus = v_minus - 2 * (v_minus . n_hat) * n_hat - mom_plus = rho_plus * v_plus - E_plus = E_minus - """ - # Grab some boundary-relevant data - dim = discr.dim - cv = split_conserved(dim, q) - actx = cv.mass.array_context - - # Grab a unit normal to the boundary - nhat = thaw(actx, discr.normal(btag)) + """Get the interior and exterior solution on the boundary.""" - # Get the interior/exterior solns + bndry_soln = self.exterior_soln(discr, q, btag, **kwargs) int_soln = discr.project("vol", btag, q) - int_cv = split_conserved(dim, int_soln) - - # Subtract out the 2*wall-normal component - # of velocity from the velocity at the wall to - # induce an equal but opposite wall-normal (reflected) wave - # preserving the tangential component - mom_normcomp = np.dot(int_cv.momentum, nhat) # wall-normal component - wnorm_mom = nhat * mom_normcomp # wall-normal mom vec - ext_mom = int_cv.momentum - 2.0 * wnorm_mom # prescribed ext momentum - - # Form the external boundary solution with the new momentum - bndry_soln = join_conserved(dim=dim, mass=int_cv.mass, - energy=int_cv.energy, - momentum=ext_mom, - species_mass=int_cv.species_mass) return TracePair(btag, interior=int_soln, exterior=bndry_soln) - def exterior_sol( - self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() - ): - """Get the interior and exterior solution on the boundary. + def exterior_soln(self, discr, q, btag, **kwargs): + """Get the exterior solution on the boundary. The exterior solution is set such that there will be vanishing flux through the boundary, preserving mass, momentum (magnitude) and @@ -210,10 +165,9 @@ def exterior_sol( return bndry_soln - def av( - self, discr, q, t=0.0, btag=BTAG_ALL, eos=IdealSingleGas() - ): - """Do artificial viscosity function.""" + def av(self, discr, q, btag, **kwargs): + """Get the exterior solution on the boundary.""" + # Grab some boundary-relevant data dim = discr.dim cv = split_conserved(dim, q) From 3a13a05a02744830b8f0b8cc5a30756a1ed67107 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Thu, 4 Feb 2021 13:29:31 -0800 Subject: [PATCH 021/385] Fix docstyle, remove unused import --- mirgecom/boundary.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index bf5272fc6..153fec29e 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -35,7 +35,6 @@ import numpy as np from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa -from mirgecom.eos import IdealSingleGas from mirgecom.euler import split_conserved, join_conserved from grudge.symbolic.primitives import TracePair @@ -121,7 +120,6 @@ class AdiabaticSlipBoundary: def boundary_pair(self, discr, q, btag, **kwargs): """Get the interior and exterior solution on the boundary.""" - bndry_soln = self.exterior_soln(discr, q, btag, **kwargs) int_soln = discr.project("vol", btag, q) @@ -167,7 +165,6 @@ def exterior_soln(self, discr, q, btag, **kwargs): def av(self, discr, q, btag, **kwargs): """Get the exterior solution on the boundary.""" - # Grab some boundary-relevant data dim = discr.dim cv = split_conserved(dim, q) From 1b2b30e199bcf53505976d6553efd5cde8df1d55 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Thu, 4 Feb 2021 16:29:24 -0800 Subject: [PATCH 022/385] Expose smoothness indicator parameters --- mirgecom/artificial_viscosity.py | 4 ++-- mirgecom/simutil.py | 11 ++++++++--- mirgecom/tag_cells.py | 13 ++++--------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 7ba1d0cb9..fc4626a96 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -68,11 +68,11 @@ def _facial_flux_q(discr, q_tpair): return discr.project(q_tpair.dd, "all_faces", flux_out) -def artificial_viscosity(discr, t, eos, boundaries, r, alpha): +def artificial_viscosity(discr, t, eos, boundaries, r, alpha, **kwargs): r"""Compute artifical viscosity for the euler equations.""" # Get smoothness indicator epsilon = np.zeros((2 + discr.dim,), dtype=object) - indicator = smoothness_indicator(r[0], discr) + indicator = smoothness_indicator(r[0], discr, **kwargs) for i in range(2 + discr.dim): epsilon[i] = indicator diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 6a138086a..c8be8b6af 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -93,7 +93,7 @@ def __init__(self, step, t, state): def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, step=0, t=0, dt=0, cfl=1.0, nstatus=-1, nviz=-1, exittol=1e-16, - constant_cfl=False, comm=None, overwrite=False): + s0=None, kappa=None, constant_cfl=False, comm=None, overwrite=False): """Check simulation health, status, viz dumps, and restart.""" # TODO: Add restart do_viz = check_step(step=step, interval=nviz) @@ -106,7 +106,8 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, dependent_vars = eos.dependent_vars(cv) from mirgecom.tag_cells import smoothness_indicator - tagedcells = smoothness_indicator(q[0], discr) + if s0 is not None and kappa is not None: + tagedcells = smoothness_indicator(q[0], discr, kappa=kappa, s0=s0) rank = 0 if comm is not None: @@ -125,13 +126,17 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, io_fields = [ ("cv", cv), ("dv", dependent_vars), - ("tagged", tagedcells) ] if exact_soln is not None: exact_list = [ ("exact_soln", expected_state), ] io_fields.extend(exact_list) + if s0 is not None and kappa is not None: + tagged_list = [ + ("tagged", tagedcells), + ] + io_fields.extend(tagged_list) from mirgecom.io import make_rank_fname, make_par_fname rank_fn = make_rank_fname(basename=vizname, rank=rank, step=step, t=t) diff --git a/mirgecom/tag_cells.py b/mirgecom/tag_cells.py index 3288ae921..20e223046 100644 --- a/mirgecom/tag_cells.py +++ b/mirgecom/tag_cells.py @@ -73,7 +73,7 @@ def compute_smoothness_indicator(): return knl -def smoothness_indicator(u, discr): +def smoothness_indicator(u, discr, kappa=1.0, s0=-6.0): """Calculate the smoothness indicator.""" assert isinstance(u, DOFArray) @@ -120,17 +120,12 @@ def get_indicator(): # Take log10 of indicator indicator = actx.np.log10(indicator + 1.0e-12) - # No special meaning to these values - # Should be exposed as tuning parameters - kappa = 0.5 - so = -7.0 - # Compute artificail viscosity percentage based on idicator and set parameters - yesnol = indicator > (so - kappa) - yesnou = indicator > (so + kappa) + yesnol = indicator > (s0 - kappa) + yesnou = indicator > (s0 + kappa) sin_indicator = actx.np.where( yesnol, - 0.5 * (1.0 + actx.np.sin(np.pi * (indicator - so) / (2.0 * kappa))), + 0.5 * (1.0 + actx.np.sin(np.pi * (indicator - s0) / (2.0 * kappa))), 0.0 * indicator, ) indicator = actx.np.where(yesnou, 1.0 + 0.0 * indicator, sin_indicator) From 0dff5e53dd2625cf91b68648b8f8abd72685dece Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Fri, 26 Feb 2021 15:09:10 -0800 Subject: [PATCH 023/385] Fixed boundary calls to self --- mirgecom/boundary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 153fec29e..a3601d080 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -61,7 +61,7 @@ def __init__(self, userfunc): def boundary_pair(self, discr, q, btag, **kwargs): """Get the interior and exterior solution on the boundary.""" - ext_soln = self.exterior_soln(self, discr, q, btag, **kwargs) + ext_soln = self.exterior_soln(discr, q, btag, **kwargs) int_soln = discr.project("vol", btag, q) return TracePair(btag, interior=int_soln, exterior=ext_soln) @@ -87,7 +87,7 @@ class DummyBoundary: def boundary_pair(self, discr, q, btag, **kwargs): """Get the interior and exterior solution on the boundary.""" - dir_soln = self.exterior_soln(self, discr, q, btag, **kwargs) + dir_soln = self.exterior_soln(discr, q, btag, **kwargs) return TracePair(btag, interior=dir_soln, exterior=dir_soln) def exterior_soln(self, discr, q, btag, **kwargs): From 1b57517c8f1edca791906a2fc919d3a32c2828d7 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Fri, 26 Feb 2021 15:18:24 -0800 Subject: [PATCH 024/385] Fixed flake8 line to long --- mirgecom/simutil.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index c8be8b6af..5b8183a5f 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -93,7 +93,8 @@ def __init__(self, step, t, state): def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, step=0, t=0, dt=0, cfl=1.0, nstatus=-1, nviz=-1, exittol=1e-16, - s0=None, kappa=None, constant_cfl=False, comm=None, overwrite=False): + s0=None, kappa=None, constant_cfl=False, comm=None, + overwrite=False): """Check simulation health, status, viz dumps, and restart.""" # TODO: Add restart do_viz = check_step(step=step, interval=nviz) From f39deb2ab389d5e3f39333246a11aae2de89e8fe Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Tue, 2 Mar 2021 12:16:25 -0800 Subject: [PATCH 025/385] Clean up AdiabaticSlipBoundary for artificial viscosity condition. --- mirgecom/boundary.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index a3601d080..e4c7d2e30 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -181,21 +181,17 @@ def av(self, discr, q, btag, **kwargs): result = np.zeros(2+dim, dtype=object) # flip signs on mass and energy + # to apply a neumann condition on q result[0] = -1.0*bndry_q.mass result[1] = -1.0*bndry_q.energy - # This needs to be removed - result[2:] = bndry_q.momentum - - # things are in the wrong order here...flip? - for i in range(dim): - tmp = np.zeros(dim, dtype=object) - for j in range(dim): - tmp[j] = bndry_q.momentum[j][i] - flip = np.dot(tmp, normal) - norm_flip = normal*flip - tmp = tmp - 2.0*norm_flip - for j in range(dim): - result[2+j][i] = -1.0*tmp[j] + # Subtract 2*wall-normal component of q + # to enforce q=0 on the wall + # flip remaining components to set a neumann condition + from pytools.obj_array import make_obj_array + q_mom_normcomp = make_obj_array( + [np.outer(normal,np.dot(bndry_q.momentum,normal))[i] for i in range(dim)] + ) + result[2:] = -1*(bndry_q.momentum-2.0*q_mom_normcomp) return(result) From 6b9a29d335b1c7b52d0b11a3a615fad93dd0d123 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Fri, 12 Mar 2021 14:30:00 -0600 Subject: [PATCH 026/385] Added inital documentation to artificial viscosity routines (#8) * Added inital documentation to artificial viscosity routines * Fix style issues * more style fixes Co-authored-by: Wyatt Hagen --- mirgecom/artificial_viscosity.py | 29 +++++++++++++++++++++++++++- mirgecom/tag_cells.py | 33 +++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 53bc06441..6ba1bde96 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -1,4 +1,31 @@ -""":mod:`mirgecom.artificial viscosity` Artificial viscocity for Euler.""" +r""":mod:`mirgecom.artificial viscosity` Artificial viscocity for Euler. + +Artificial viscosity for the Euler Equations: + +.. math:: + + \partial_t \mathbf{Q} = \nabla\cdot{\varepsilon\nabla\mathbf{Q}} + +where: + +- state $\mathbf{Q} = [\rho, \rho{E}, \rho\vec{V}, \rho{Y}_\alpha]$ +- artifical viscosity coefficient $\varepsilon$ + +To evalutate the second order derivative the problem is recast as a set of first + order problems: + +.. math:: + + \partial_t \mathbf{Q} = \nabla\cdot{\mathbf{R}} + \mathbf{R} = \varepsilon\nabla\mathbf{Q} + +where $\mathbf{R}$ is an intermediate variable. + +RHS Evaluation +^^^^^^^^^^^^^^ + +.. autofunction:: artificial_viscosity +""" __copyright__ = """ Copyright (C) 2020 University of Illinois Board of Trustees diff --git a/mirgecom/tag_cells.py b/mirgecom/tag_cells.py index 20e223046..a55abaacd 100644 --- a/mirgecom/tag_cells.py +++ b/mirgecom/tag_cells.py @@ -1,13 +1,44 @@ r""":mod:`mirgecom.tag_cells` Compute smoothness indicator. -Perssons smoothness indicator: +Evalutes the smoothness indicator of Persson: .. math:: S_e = \frac{\langle u_{N_p} - u_{N_{p-1}}, u_{N_p} - u_{N_{p-1}}\rangle_e}{\langle u_{N_p}, u_{N_p} \rangle_e} +where: +- $S_e$ is the smoothness indicator +- $u_{N_p}$ is the modal representation of the solution at the current polynomial + order +- $u_{N_{p-1}}$ is the truncated modal represention to the polynomial order $p-1$ +- The $L_2$ inner product on an element is denoted $\langle \cdot,\cdot \rangle_e$ + +The elementwise viscoisty is then calculated: + +.. math:: + + \varepsilon_e = \varepsilon_0 + \begin{cases} + 0, & s_e < s_0 - \kappa \\ + \frac{1}{2}\left( 1 + \sin \frac{\pi(s_e - s_0)}{2 \kappa} \right ), + & s_0-\kappa \le s_e \le s_0+\kappa \\ + 1, & s_e > s_0+\kappa + \end{cases} + +where: +- $\varepsilon_e$ is the element viscosity +- $\varepsilon_0 ~\sim h/p$ is a reference viscosity +- $s_e = \log_{10}{S_e} \sim 1/p^4$ is the smoothness indicator +- $s_0$ is a reference smoothness value +- $\kappa$ controls the width of the transition between 0 to $\varepsilon_0$ + +Smoothness Indicator Evaluation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. autofunction:: smoothness_indicator """ + __copyright__ = """ Copyright (C) 2020 University of Illinois Board of Trustees """ From d17ea87e57a0d60e7dd75460d81f4b72a25c3ca2 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Fri, 12 Mar 2021 15:09:00 -0800 Subject: [PATCH 027/385] fixed flake8 error --- mirgecom/boundary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index e4c7d2e30..a407f72d7 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -190,7 +190,7 @@ def av(self, discr, q, btag, **kwargs): # flip remaining components to set a neumann condition from pytools.obj_array import make_obj_array q_mom_normcomp = make_obj_array( - [np.outer(normal,np.dot(bndry_q.momentum,normal))[i] for i in range(dim)] + [np.outer(normal, np.dot(bndry_q.momentum, normal))[i] for i in range(dim)] ) result[2:] = -1*(bndry_q.momentum-2.0*q_mom_normcomp) From 100b1c3f708313ac80830bc5fef73e2e00d36f29 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Fri, 12 Mar 2021 15:14:33 -0800 Subject: [PATCH 028/385] Fixed too long line --- mirgecom/boundary.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index a407f72d7..84c471d45 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -190,7 +190,8 @@ def av(self, discr, q, btag, **kwargs): # flip remaining components to set a neumann condition from pytools.obj_array import make_obj_array q_mom_normcomp = make_obj_array( - [np.outer(normal, np.dot(bndry_q.momentum, normal))[i] for i in range(dim)] + [np.outer(normal, np.dot(bndry_q.momentum, normal))[i] + for i in range(dim)] ) result[2:] = -1*(bndry_q.momentum-2.0*q_mom_normcomp) From 9d0f8edd7433c15ed80fab33d086e9f769236f9b Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Fri, 12 Mar 2021 17:23:46 -0600 Subject: [PATCH 029/385] Added tests to artificial viscosity routines. (#9) * Added tests to artificial viscosity routines. * Fixed flake8 errors. * Fixed error in getting basis polynomials * Fixed style errors Co-authored-by: Wyatt Hagen --- test/test_artificial_viscosity.py | 200 ++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 test/test_artificial_viscosity.py diff --git a/test/test_artificial_viscosity.py b/test/test_artificial_viscosity.py new file mode 100644 index 000000000..972cfc1b6 --- /dev/null +++ b/test/test_artificial_viscosity.py @@ -0,0 +1,200 @@ +"""Test the artificial viscosity functions.""" + +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import logging +import numpy as np +import pyopencl as cl +import pytest +from meshmode.array_context import PyOpenCLArrayContext +from meshmode.dof_array import thaw +from meshmode.mesh import BTAG_ALL + +from mirgecom.tag_cells import smoothness_indicator +from mirgecom.artificial_viscosity import artificial_viscosity +from mirgecom.boundary import DummyBoundary +from grudge.eager import EagerDGDiscretization +from pytools.obj_array import flat_obj_array +from pyopencl.tools import ( # noqa + pytest_generate_tests_for_pyopencl as pytest_generate_tests, +) + +logger = logging.getLogger(__name__) + + +@pytest.mark.parametrize("dim", [1, 2, 3]) +@pytest.mark.parametrize("order", [1, 5]) +def test_tag_cells(ctx_factory, dim, order): + """Test tag_cells. + + Tests that the cells tagging properly tags cells + given a prescirbed solutions. + """ + cl_ctx = ctx_factory() + queue = cl.CommandQueue(cl_ctx) + actx = PyOpenCLArrayContext(queue) + + nel_1d = 2 + tolerance = 1.e-16 + + def norm_indicator(expected, discr, soln, **kwargs): + return(discr.norm(expected-smoothness_indicator(soln, discr, **kwargs), + np.inf)) + + from meshmode.mesh.generation import generate_regular_rect_mesh + + mesh = generate_regular_rect_mesh( + a=(-1.0, )*dim, b=(1.0, )*dim, n=(nel_1d, ) * dim + ) + + discr = EagerDGDiscretization(actx, mesh, order=order) + nodes = thaw(actx, discr.nodes()) + nele = mesh.nelements + zeros = 0.0*nodes[0] + + # test jump discontinuity + soln = actx.np.where(nodes[0] > 0.0+zeros, 1.0+zeros, zeros) + err = norm_indicator(1.0, discr, soln) + + assert err < tolerance, "Jump discontinuity should trigger indicator (1.0)" + + # get meshmode polynomials + group = discr.discr_from_dd("vol").groups[0] + basis = group.basis() # only one group + unit_nodes = group.unit_nodes + modes = group.mode_ids() + order = group.order + + # loop over modes and check smoothness + for i, mode in enumerate(modes): + ele_soln = basis[i](unit_nodes) + soln[0].set(np.tile(ele_soln, (nele, 1))) + if sum(mode) == order: + expected = 1.0 + else: + expected = 0.0 + err = norm_indicator(expected, discr, soln) + assert err < tolerance, "Only highest modes should trigger indicator (1.0)" + + # Test s0 + s0 = -1. + eps = 1.0e-6 + + phi_n_p = np.sqrt(np.power(10, s0)) + phi_n_pm1 = np.sqrt(1. - np.power(10, s0)) + + # pick a polynomial of order n_p, n_p-1 + n_p = np.array(np.nonzero((np.sum(modes, axis=1) == order))).flat[0] + n_pm1 = np.array(np.nonzero((np.sum(modes, axis=1) == order-1))).flat[0] + + # create test soln perturbed around + # Solution above s0 + ele_soln = ((phi_n_p+eps)*basis[n_p](unit_nodes) + + phi_n_pm1*basis[n_pm1](unit_nodes)) + soln[0].set(np.tile(ele_soln, (nele, 1))) + err = norm_indicator(1.0, discr, soln, s0=s0, kappa=0.0) + assert err < tolerance, ( + "A function with an indicator >s0 should trigger indicator") + + # Solution below s0 + ele_soln = ((phi_n_p-eps)*basis[n_p](unit_nodes) + + phi_n_pm1*basis[n_pm1](unit_nodes)) + soln[0].set(np.tile(ele_soln, (nele, 1))) + err = norm_indicator(0.0, discr, soln, s0=s0, kappa=0.0) + assert err < tolerance, ( + "A function with an indicator tolerance, "s_e>s_0-kappa should trigger indicator" + + # lower bound + err = norm_indicator(1.0, discr, soln, s0=s0-(kappa+shift), kappa=kappa) + assert err < tolerance, "s_e>s_0+kappa should fully trigger indicator (1.0)" + err = norm_indicator(1.0, discr, soln, s0=s0-(kappa-shift), kappa=kappa) + assert err > tolerance, "s_e Date: Fri, 12 Mar 2021 16:13:46 -0800 Subject: [PATCH 030/385] Fixed missed merge conflict. --- mirgecom/simutil.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 4b6fad0a7..a6227f0bf 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -132,16 +132,13 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, ("exact_soln", expected_state), ] io_fields.extend(exact_list) -<<<<<<< HEAD if s0 is not None and kappa is not None: tagged_list = [ ("tagged", tagedcells), ] io_fields.extend(tagged_list) -======= if viz_fields is not None: io_fields.extend(viz_fields) ->>>>>>> main from mirgecom.io import make_rank_fname, make_par_fname rank_fn = make_rank_fname(basename=vizname, rank=rank, step=step, t=t) From 34b96bffb1991712a07431cac20ff0a0723264ee Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Sat, 13 Mar 2021 18:26:12 -0800 Subject: [PATCH 031/385] Fixed errors in dobule mach initialization. --- mirgecom/initializers.py | 31 ++----------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 91a0660fa..b32d1a734 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -298,7 +298,7 @@ def __init__( self._dim = dim self._us = us - def __call__(self, t, x_vec, eos=IdealSingleGas()): + def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): """ Create the 1D Sod's shock solution at locations *x_vec*. @@ -325,28 +325,6 @@ def __call__(self, t, x_vec, eos=IdealSingleGas()): us = zeros + self._us t = zeros + t - # Mach 10 - # rhol = zeros + 8.0 - # rhor = zeros + 1.4 - - # ul = zeros + 8.25*np.cos(np.pi/6.0) - # ur = zeros + 0.0 - # vl = zeros - 8.25*np.sin(np.pi/6.0) - # vr = zeros + 0.0 - # rhoel = zeros + gmn1 * 116.5 - # rhoer = zeros + gmn1 * 1.0 - - # Mach 2.0 - # rhol = zeros + 2.666666*1.4 - # rhor = zeros + 1.4 - - # ul = zeros + 1.25*np.cos(np.pi/6.0) - # ur = zeros + 0.0 - # vl = zeros - 1.25*np.sin(np.pi/6.0) - # vr = zeros + 0.0 - # rhoel = zeros + gmn1 * 4.5 - # rhoer = zeros + gmn1 * 1.0 - # Mach 4.0 rhol = zeros + 4.57142857*1.4 rhor = zeros + 1.4 @@ -358,11 +336,6 @@ def __call__(self, t, x_vec, eos=IdealSingleGas()): rhoel = zeros + gmn1 * 18.5 rhoer = zeros + gmn1 * 1.0 - # yesno = x_rel > (x0 + y_rel/np.sqrt(3.0) + 2.0*us*t/np.sqrt(3.0)) - # mass = actx.np.where(yesno, rhor, rhol) - # rhoe = actx.np.where(yesno, rhoer, rhoel) - # u = actx.np.where(yesno, ur, ul) - # v = actx.np.where(yesno, vr, vl) xinter = (x0 + y_rel/np.sqrt(3.0) + 2.0*us*t/np.sqrt(3.0)) sigma = 0.05 xtanh = 1.0/sigma*(x_rel-xinter) @@ -375,7 +348,7 @@ def __call__(self, t, x_vec, eos=IdealSingleGas()): rhov = mass*v energy = rhoe + 0.5*mass*(u*u + v*v) - return flat_obj_array(mass, energy, rhou, rhov) + return join_conserved(dim=self._dim, mass=mass, energy=energy, momentum=make_obj_array([u,v])) class Lump: From f52a165085cf9d2965a2aa6653922c9c3b1df8c4 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen Date: Sat, 13 Mar 2021 19:58:21 -0800 Subject: [PATCH 032/385] Fixed error in initialization. --- mirgecom/initializers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index b32d1a734..8c8bd87d0 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -348,7 +348,8 @@ def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): rhov = mass*v energy = rhoe + 0.5*mass*(u*u + v*v) - return join_conserved(dim=self._dim, mass=mass, energy=energy, momentum=make_obj_array([u,v])) + return join_conserved(dim=self._dim, mass=mass, energy=energy, + momentum=make_obj_array([rhou, rhov])) class Lump: From c8858e4c635616146361cc996ee55c9d75cb8d6e Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Sun, 14 Mar 2021 11:57:41 -0700 Subject: [PATCH 033/385] Cleaned up artificial viscosity routines. --- mirgecom/artificial_viscosity.py | 49 ++------------------------------ mirgecom/tag_cells.py | 7 +---- 2 files changed, 4 insertions(+), 52 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 6ba1bde96..fe6cb1a3a 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -51,8 +51,6 @@ THE SOFTWARE. """ -# from dataclasses import dataclass - import numpy as np from pytools.obj_array import ( obj_array_vectorize, @@ -62,7 +60,6 @@ from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.eager import interior_trace_pair, cross_rank_trace_pairs from grudge.symbolic.primitives import TracePair -# from mirgecom.euler import split_conserved, join_conserved from mirgecom.tag_cells import smoothness_indicator @@ -76,11 +73,6 @@ def _facial_flux_r(discr, q_tpair): flux_out = flux_dis * normal - # Can't do it here... "obj arrays not allowed on compute device" - # def flux_calc(flux): - # return (flux * normal) - # flux_out = obj_array_vectorize(flux_calc,flux_dis) - return discr.project(q_tpair.dd, "all_faces", flux_out) @@ -98,51 +90,23 @@ def _facial_flux_q(discr, q_tpair): def artificial_viscosity(discr, t, eos, boundaries, r, alpha, **kwargs): r"""Compute artifical viscosity for the euler equations.""" # Get smoothness indicator - epsilon = np.zeros((2 + discr.dim,), dtype=object) indicator = smoothness_indicator(r[0], discr, **kwargs) - for i in range(2 + discr.dim): - epsilon[i] = indicator - - # compute dissapation flux - # Cannot call weak_grad on obj of nd arrays - # use obj_array_vectorize as work around dflux_r = obj_array_vectorize(discr.weak_grad, r) - # interior face flux - - # Doesn't work: something related to obj on compute device - # Not 100% on reason - # qin = interior_trace_pair(discr,r) - # iff_r = _facial_flux(discr,q_tpair=qin) - - # Work around? def my_facialflux_r_interior(q): qin = interior_trace_pair(discr, q) return _facial_flux_r(discr, q_tpair=qin) iff_r = obj_array_vectorize(my_facialflux_r_interior, r) - # partition boundaries flux - # flux across partition boundaries def my_facialflux_r_partition(q): qin = cross_rank_trace_pairs(discr, q) return sum(_facial_flux_r(discr, q_tpair=part_pair) for part_pair in qin) pbf_r = obj_array_vectorize(my_facialflux_r_partition, r) - # True boundary implementation - # Okay, not sure about this... - # What I am attempting: - # 1. Loop through all the boundaries - # 2. Define a function my_TP that performes the trace pair for the given - # boundary given a solution variable - # 3. Get the external solution from the boundary routine - # 4. Get hte projected internal solution - # 5. Compute the boundary flux as a sum over boundaries, using the - # obj_array_vectorize to pass each solution variable one at a time - # DO I really need to do this like this? - dbf_r = 0.0 * iff_r + dbf_r = np.zeros_like(iff_r) for btag in boundaries: def my_facialflux_r_boundary(sol_ext, sol_int): @@ -160,18 +124,12 @@ def my_facialflux_r_boundary(sol_ext, sol_int): ) # Compute q, half way done! - # q = discr.inverse_mass(-alpha*(dflux_r-discr.face_mass(iff_r + pbf_r + dbf_r))) q = discr.inverse_mass( - -alpha * epsilon * (dflux_r - discr.face_mass(iff_r + pbf_r + dbf_r)) + -alpha * indicator * (dflux_r - discr.face_mass(iff_r + pbf_r + dbf_r)) ) - # flux of q - - # Again we need to vectorize - # q is a object array of object arrays (dim,) of DOFArrays (?) dflux_q = obj_array_vectorize(discr.weak_div, q) - # interior face flux of q def my_facialflux_q_interior(q): qin = interior_trace_pair(discr, q) iff_q = _facial_flux_q(discr, q_tpair=qin) @@ -179,14 +137,13 @@ def my_facialflux_q_interior(q): iff_q = obj_array_vectorize(my_facialflux_q_interior, q) - # flux across partition boundaries def my_facialflux_q_partition(q): qin = cross_rank_trace_pairs(discr, q) return sum(_facial_flux_q(discr, q_tpair=part_pair) for part_pair in qin) pbf_q = obj_array_vectorize(my_facialflux_q_partition, q) - dbf_q = 0.0 * iff_q + dbf_q = np.zeros_like(iff_q) for btag in boundaries: def my_facialflux_q_boundary(sol_ext, sol_int): diff --git a/mirgecom/tag_cells.py b/mirgecom/tag_cells.py index a55abaacd..4167214c9 100644 --- a/mirgecom/tag_cells.py +++ b/mirgecom/tag_cells.py @@ -100,7 +100,6 @@ def compute_smoothness_indicator(): " vec[iel,j]*vec[iel,j]+1.0e-12/ndiscr_nodes_in)", name="smooth_comp", ) - # knl = lp.tag_array_axes(knl, "vec", "stride:auto,stride:auto") return knl @@ -108,11 +107,9 @@ def smoothness_indicator(u, discr, kappa=1.0, s0=-6.0): """Calculate the smoothness indicator.""" assert isinstance(u, DOFArray) - # #@memoize_in(u.array_context, (smoothness_indicator, "get_kernel")) def get_kernel(): return linear_operator_kernel() - # #@memoize_in(u.array_context, (smoothness_indicator, "get_indicator")) def get_indicator(): return compute_smoothness_indicator() @@ -147,11 +144,9 @@ def get_indicator(): vec=uhat[group.index], modes=actx.from_numpy(highest_mode), ) - - # Take log10 of indicator indicator = actx.np.log10(indicator + 1.0e-12) - # Compute artificail viscosity percentage based on idicator and set parameters + # Compute artificial viscosity percentage based on indicator and set parameters yesnol = indicator > (s0 - kappa) yesnou = indicator > (s0 + kappa) sin_indicator = actx.np.where( From 1afa3d78d52979d779e389171b582dac165f1973 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Sun, 14 Mar 2021 14:09:04 -0700 Subject: [PATCH 034/385] Fixed errors in calling gmsh --- examples/doublemach-mpi.py | 81 +++++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 22 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index d23adb80f..f775b54f4 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -54,6 +54,51 @@ logger = logging.getLogger(__name__) +def get_doublemach_mesh(): + """Generate or import a grid using `gmsh`. + + Input required: + doubleMach.msh (read existing mesh + + This routine will generate a new grid if it does + not find the grid file (doubleMach.msh). + """ + from meshmode.mesh.io import ( + read_gmsh, + generate_gmsh, + ScriptWithFilesSource, + ScriptSource, + ) + import os + meshfile="doubleMach.msh" + if os.path.exists(meshfile) is False: + mesh = generate_gmsh( + ScriptSource(""" + x0=1.0/6.0; + setsize=0.025; + Point(1) = {0, 0, 0, setsize}; + Point(2) = {x0,0, 0, setsize}; + Point(3) = {4, 0, 0, setsize}; + Point(4) = {4, 1, 0, setsize}; + Point(5) = {0, 1, 0, setsize}; + Line(1) = {1, 2}; + Line(2) = {2, 3}; + Line(5) = {3, 4}; + Line(6) = {4, 5}; + Line(7) = {5, 1}; + Line Loop(8) = {-5, -6, -7, -1, -2}; + Plane Surface(8) = {8}; + Physical Surface('domain') = {8}; + Physical Curve('ic1') = {6}; + Physical Curve('ic2') = {7}; + Physical Curve('ic3') = {1}; + Physical Curve('wall') = {2}; + Physical Curve('out') = {5}; + ""","geo"), force_ambient_dim=2, dimensions=2, target_unit="M") + else: + mesh = read_gmsh(meshfile,force_ambient_dim=2) + + return mesh @mpi_entry_point def main(ctx_factory=cl.create_some_context): @@ -65,19 +110,14 @@ def main(ctx_factory=cl.create_some_context): ) dim = 2 - # nel = (40, 10) order = 3 - # tolerate large errors; case is unstable - exittol = 2.0 # .2 - t_final = 0.0001 + t_final = 1.0e-2 current_cfl = 0.1 - current_dt = 0.000001 + current_dt = 1.0e-4 current_t = 0 eos = IdealSingleGas() initializer = DoubleMachReflection(dim) - # initializer = SodShock1D(dim,x0=0.5) - casename = "sod1d" - # boundaries = {BTAG_ALL: AdiabaticSlipBoundary()} + casename = "doubleMach" from grudge import sym boundaries = { @@ -94,16 +134,16 @@ def main(ctx_factory=cl.create_some_context): checkpoint_t = current_t current_step = 0 timestepper = rk4_step - # box_ll = (0.0, 0.0) - # box_ur = (4.0, 1.0) + s0 = -6.0 + kappa = 1.0 + alpha = 2.0e-2 from mpi4py import MPI comm = MPI.COMM_WORLD - rank = comm.Get_rank() + rank = comm.Get_rank - from meshmode.mesh.io import read_gmsh - gen_grid = partial(read_gmsh, "doubleMach2.msh", force_ambient_dim=2) + gen_grid = partial(get_doublemach_mesh) local_mesh, global_nelements = create_parallel_grid(comm, gen_grid) @@ -112,10 +152,8 @@ def main(ctx_factory=cl.create_some_context): discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) nodes = thaw(actx, discr.nodes()) - current_state = initializer(0, nodes) + current_state = initializer(nodes) - # visualizer = make_visualizer(discr, discr.order + 3 - # if discr.dim == 2 else discr.order) visualizer = make_visualizer(discr, discr.order if discr.dim == 2 else discr.order) initname = initializer.__class__.__name__ @@ -153,12 +191,9 @@ def my_rhs(t, state): return inviscid_operator( discr, q=state, t=t, boundaries=boundaries, eos=eos ) + artificial_viscosity( - discr, t=t, r=state, eos=eos, boundaries=boundaries, alpha=2.0e-2 + discr, t=t, r=state, eos=eos, boundaries=boundaries, alpha=alpha, + s0=s0, kappa=kappa ) - # return ( - # inviscid_operator(discr, q=state, t=t,boundaries=boundaries, eos=eos) - # + artificial_viscosity(discr, r=state, eos=eos, boundaries=boundaries, - # alpha=1.0e-3) def my_checkpoint(step, t, dt, state): return sim_checkpoint( @@ -172,9 +207,11 @@ def my_checkpoint(step, t, dt, state): dt=dt, nstatus=nstatus, nviz=nviz, - exittol=exittol, constant_cfl=constant_cfl, comm=comm, + s0=s0, + kappa=kappa, + overwrite=True, ) try: From a404a62d6db4d6747c59dad8d29dd82228be8297 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Sun, 14 Mar 2021 14:38:00 -0700 Subject: [PATCH 035/385] Add gmsh as requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2af096ad4..4349c925b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ pyvisfile pymetis importlib-resources psutil +gmsh # The following packages will be git cloned by emirge: --editable git+https://github.com/inducer/pymbolic.git#egg=pymbolic From c1cd4792d1e6ef1d262b5e0d320e7fb9fbea55b5 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Sun, 14 Mar 2021 22:02:01 -0700 Subject: [PATCH 036/385] Fix some style issues and add temporary fix for Issue #280 --- examples/doublemach-mpi.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index f775b54f4..6e77514bc 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -54,6 +54,7 @@ logger = logging.getLogger(__name__) + def get_doublemach_mesh(): """Generate or import a grid using `gmsh`. @@ -66,11 +67,10 @@ def get_doublemach_mesh(): from meshmode.mesh.io import ( read_gmsh, generate_gmsh, - ScriptWithFilesSource, ScriptSource, ) import os - meshfile="doubleMach.msh" + meshfile = "doubleMach.msh" if os.path.exists(meshfile) is False: mesh = generate_gmsh( ScriptSource(""" @@ -94,12 +94,13 @@ def get_doublemach_mesh(): Physical Curve('ic3') = {1}; Physical Curve('wall') = {2}; Physical Curve('out') = {5}; - ""","geo"), force_ambient_dim=2, dimensions=2, target_unit="M") + """, "geo"), force_ambient_dim=2, dimensions=2, target_unit="M") else: - mesh = read_gmsh(meshfile,force_ambient_dim=2) + mesh = read_gmsh(meshfile, force_ambient_dim=2) return mesh + @mpi_entry_point def main(ctx_factory=cl.create_some_context): """Drive the example.""" @@ -151,6 +152,17 @@ def main(ctx_factory=cl.create_some_context): discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) + + # Temporary fix for Issue #280 + local_boundaries = {} + for btag in boundaries: + bnd_discr = discr.discr_from_dd(btag) + bnd_nodes = thaw(actx, bnd_discr.nodes()) + if bnd_nodes[0][0].shape[0] > 0: + local_boundaries[btag] = boundaries[btag] + boundaries = local_boundaries + # End fix + nodes = thaw(actx, discr.nodes()) current_state = initializer(nodes) From 8b580e5014dd25f70d86529b1ba496de7752ad2a Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 15 Mar 2021 13:34:42 -0500 Subject: [PATCH 037/385] Fix documentation grammer. Co-authored-by: Mike Campbell --- test/test_artificial_viscosity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_artificial_viscosity.py b/test/test_artificial_viscosity.py index 972cfc1b6..822a7eaa9 100644 --- a/test/test_artificial_viscosity.py +++ b/test/test_artificial_viscosity.py @@ -51,7 +51,7 @@ def test_tag_cells(ctx_factory, dim, order): """Test tag_cells. Tests that the cells tagging properly tags cells - given a prescirbed solutions. + given prescribed solutions. """ cl_ctx = ctx_factory() queue = cl.CommandQueue(cl_ctx) From 40364ac41ce999e79573e5980e6f671e8d20851a Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 15 Mar 2021 12:06:34 -0700 Subject: [PATCH 038/385] Renamed shock parameters to be more descriptive. --- mirgecom/initializers.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 8c8bd87d0..593c10e49 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -281,7 +281,7 @@ class DoubleMachReflection: """ def __init__( - self, dim=2, x0=1.0/6.0, us=4.0 + self, dim=2, shock_location=1.0/6.0, shock_speed=4.0 ): """Initialize initial condition options. @@ -289,14 +289,14 @@ def __init__( ---------- dim: int dimension of domain, must be 2 - x0: float + shock_location: float location of shock - us: float - shock speed + shock_speed: float + shock speed, Mach number """ - self._x0 = x0 + self._shock_location = shock_location self._dim = dim - self._us = us + self._shock_speed = shock_speed def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): """ @@ -321,8 +321,8 @@ def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): zeros = 0*x_rel - x0 = zeros + self._x0 - us = zeros + self._us + shock_location = zeros + self._shock_location + shock_speed = zeros + self._shock_speed t = zeros + t # Mach 4.0 @@ -336,7 +336,8 @@ def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): rhoel = zeros + gmn1 * 18.5 rhoer = zeros + gmn1 * 1.0 - xinter = (x0 + y_rel/np.sqrt(3.0) + 2.0*us*t/np.sqrt(3.0)) + xinter = (shock_location + y_rel/np.sqrt(3.0) + + 2.0*shock_speed*t/np.sqrt(3.0)) sigma = 0.05 xtanh = 1.0/sigma*(x_rel-xinter) mass = rhol/2.0*(actx.np.tanh(-xtanh)+1.0)+rhor/2.0*(actx.np.tanh(xtanh)+1.0) From af24249e6df0c37ea26e9748ef581b05f81bcf6f Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 15 Mar 2021 12:40:57 -0700 Subject: [PATCH 039/385] Added normal shock relations calculator to DoubleMahcReflection case. --- mirgecom/initializers.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 593c10e49..f9510c349 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -314,6 +314,7 @@ def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): assert self._dim == 2, "only defined for dim=2" gm1 = eos.gamma() - 1.0 + gp1 = eos.gamma() + 1.0 gmn1 = 1.0 / gm1 x_rel = x_vec[0] y_rel = x_vec[1] @@ -325,15 +326,19 @@ def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): shock_speed = zeros + self._shock_speed t = zeros + t - # Mach 4.0 - rhol = zeros + 4.57142857*1.4 - rhor = zeros + 1.4 + # Normal Shock Relations + shock_speed_2 = self._shock_speed * self._shock_speed + rho_jump = gp1 * shock_speed_2 / (gm1 * shock_speed_2 + 2.) + p_jump = (2. * eos.gamma() * shock_speed_2 - gm1) / gp1 + up = 2. * (shock_speed_2 - 1.) / (gp1 * self._shock_speed) - ul = zeros + 3.125*np.cos(np.pi/6.0) + rhol = zeros + eos.gamma() * rho_jump + rhor = zeros + eos.gamma() + ul = zeros + up * np.cos(np.pi/6.0) ur = zeros + 0.0 - vl = zeros - 3.125*np.sin(np.pi/6.0) + vl = zeros - up * np.sin(np.pi/6.0) vr = zeros + 0.0 - rhoel = zeros + gmn1 * 18.5 + rhoel = zeros + gmn1 * p_jump rhoer = zeros + gmn1 * 1.0 xinter = (shock_location + y_rel/np.sqrt(3.0) From 8a50f22a38d468b311358214e4791b4cab9f47b5 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 15 Mar 2021 12:52:03 -0700 Subject: [PATCH 040/385] Remove linter changes for not my file --- examples/heat-source-mpi.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/heat-source-mpi.py b/examples/heat-source-mpi.py index 8150305fc..cca028e79 100644 --- a/examples/heat-source-mpi.py +++ b/examples/heat-source-mpi.py @@ -1,5 +1,3 @@ -"""Demonstrate heat source.""" - __copyright__ = "Copyright (C) 2020 University of Illinois Board of Trustees" __license__ = """ From 77b02639eb730b820de6dd70f405f58451b2e5fb Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 15 Mar 2021 14:49:16 -0700 Subject: [PATCH 041/385] use viz_fields to visualize tagged cells. --- examples/doublemach-mpi.py | 13 ++++++++----- mirgecom/simutil.py | 13 ++----------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 6e77514bc..2bd1c719e 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -35,8 +35,9 @@ from grudge.shortcuts import make_visualizer -from mirgecom.euler import inviscid_operator +from mirgecom.euler import inviscid_operator, split_conserved from mirgecom.artificial_viscosity import artificial_viscosity +from mirgecom.tag_cells import smoothness_indicator from mirgecom.simutil import ( inviscid_sim_timestep, sim_checkpoint, @@ -59,7 +60,7 @@ def get_doublemach_mesh(): """Generate or import a grid using `gmsh`. Input required: - doubleMach.msh (read existing mesh + doubleMach.msh (read existing mesh) This routine will generate a new grid if it does not find the grid file (doubleMach.msh). @@ -203,11 +204,14 @@ def my_rhs(t, state): return inviscid_operator( discr, q=state, t=t, boundaries=boundaries, eos=eos ) + artificial_viscosity( - discr, t=t, r=state, eos=eos, boundaries=boundaries, alpha=alpha, + discr, t=t, r=state, boundaries=boundaries, alpha=alpha, eos=eos, s0=s0, kappa=kappa ) def my_checkpoint(step, t, dt, state): + cv = split_conserved(dim, state) + tagged_cells = smoothness_indicator(discr, cv.mass, s0=s0, kappa=kappa) + viz_fields = [("tagged cells", tagged_cells)] return sim_checkpoint( discr, visualizer, @@ -221,8 +225,7 @@ def my_checkpoint(step, t, dt, state): nviz=nviz, constant_cfl=constant_cfl, comm=comm, - s0=s0, - kappa=kappa, + viz_fields=viz_fields, overwrite=True, ) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index a6227f0bf..0347ec5db 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -93,8 +93,8 @@ def __init__(self, step, t, state): def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, step=0, t=0, dt=0, cfl=1.0, nstatus=-1, nviz=-1, exittol=1e-16, - s0=None, kappa=None, constant_cfl=False, comm=None, - viz_fields=None, overwrite=False, vis_timer=None): + constant_cfl=False, comm=None, viz_fields=None, overwrite=False, + vis_timer=None): """Check simulation health, status, viz dumps, and restart.""" do_viz = check_step(step=step, interval=nviz) do_status = check_step(step=step, interval=nstatus) @@ -105,10 +105,6 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, cv = split_conserved(discr.dim, q) dependent_vars = eos.dependent_vars(cv) - from mirgecom.tag_cells import smoothness_indicator - if s0 is not None and kappa is not None: - tagedcells = smoothness_indicator(q[0], discr, kappa=kappa, s0=s0) - rank = 0 if comm is not None: rank = comm.Get_rank() @@ -132,11 +128,6 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, ("exact_soln", expected_state), ] io_fields.extend(exact_list) - if s0 is not None and kappa is not None: - tagged_list = [ - ("tagged", tagedcells), - ] - io_fields.extend(tagged_list) if viz_fields is not None: io_fields.extend(viz_fields) From 218cb523e501a27c1aabb6ec56899efb9975a816 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 15 Mar 2021 14:54:09 -0700 Subject: [PATCH 042/385] Added interface parameter documentation. --- mirgecom/artificial_viscosity.py | 54 ++++++++++++++++++++++++++------ mirgecom/tag_cells.py | 30 +++++++++++++++--- 2 files changed, 70 insertions(+), 14 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index fe6cb1a3a..7b9f4df0c 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -4,11 +4,11 @@ .. math:: - \partial_t \mathbf{Q} = \nabla\cdot{\varepsilon\nabla\mathbf{Q}} + \partial_t \mathbf{R} = \nabla\cdot{\varepsilon\nabla\mathbf{R}} where: -- state $\mathbf{Q} = [\rho, \rho{E}, \rho\vec{V}, \rho{Y}_\alpha]$ +- state $\mathbf{R} = [\rho, \rho{E}, \rho\vec{V}, \rho{Y}_\alpha]$ - artifical viscosity coefficient $\varepsilon$ To evalutate the second order derivative the problem is recast as a set of first @@ -16,10 +16,10 @@ .. math:: - \partial_t \mathbf{Q} = \nabla\cdot{\mathbf{R}} - \mathbf{R} = \varepsilon\nabla\mathbf{Q} + \partial_t \mathbf{R} = \nabla\cdot{\mathbf{Q}} + \mathbf{Q} = \varepsilon\nabla\mathbf{R} -where $\mathbf{R}$ is an intermediate variable. +where $\mathbf{Q}$ is an intermediate variable. RHS Evaluation ^^^^^^^^^^^^^^ @@ -88,9 +88,45 @@ def _facial_flux_q(discr, q_tpair): def artificial_viscosity(discr, t, eos, boundaries, r, alpha, **kwargs): - r"""Compute artifical viscosity for the euler equations.""" + r"""Compute artifical viscosity for the euler equations. + + Calculates + ---------- + numpy.ndarray + The right-hand-side for artificial viscosity for the euler equations. + + .. math:: + + \dot{\nabla\cdot{\varepsilon\nabla\mathbf{R}}} + + Parameters + ---------- + r + State array which expects the quantity to be limited on to be listed + first in the array. For the Euler equations this could be the canonical + conserved variables (mass, energy, mometum) for the fluid along with a + vector of species masses for multi-component fluids. + + boundaries + Dicitionary of boundary functions, one for each valid boundary tag + + t + Time + + alpha + The maximum artifical viscosity coeffiecent to be applied + + eos: mirgecom.eos.GasEOS + Only used as a pass through to the boundary conditions. + + Returns + ------- + numpy.ndarray + Agglomerated object array of DOF Arrays representing the RHS associated + with the artificial viscosity application. + """ # Get smoothness indicator - indicator = smoothness_indicator(r[0], discr, **kwargs) + indicator = smoothness_indicator(discr, r[0], **kwargs) dflux_r = obj_array_vectorize(discr.weak_grad, r) @@ -117,7 +153,7 @@ def my_facialflux_r_boundary(sol_ext, sol_int): ) return _facial_flux_r(discr, q_tpair=q_tpair) - r_ext = boundaries[btag].exterior_soln(discr, eos=eos, btag=btag, t=t, q=r) + r_ext = boundaries[btag].exterior_soln(discr, btag=btag, t=t, q=r, eos=eos) r_int = discr.project("vol", btag, r) dbf_r = dbf_r + obj_array_vectorize_n_args( my_facialflux_r_boundary, r_ext, r_int @@ -150,7 +186,7 @@ def my_facialflux_q_boundary(sol_ext, sol_int): q_tpair = TracePair(btag, interior=sol_int, exterior=sol_ext) return _facial_flux_q(discr, q_tpair=q_tpair) - q_ext = boundaries[btag].av(discr, eos=eos, btag=btag, t=t, q=q) + q_ext = boundaries[btag].av(discr, btag=btag, t=t, q=q, eos=eos) q_int = discr.project("vol", btag, q) dbf_q = dbf_q + obj_array_vectorize_n_args( my_facialflux_q_boundary, q_ext, q_int diff --git a/mirgecom/tag_cells.py b/mirgecom/tag_cells.py index 4167214c9..539cbf97c 100644 --- a/mirgecom/tag_cells.py +++ b/mirgecom/tag_cells.py @@ -18,7 +18,7 @@ .. math:: - \varepsilon_e = \varepsilon_0 + \varepsilon_e = \begin{cases} 0, & s_e < s_0 - \kappa \\ \frac{1}{2}\left( 1 + \sin \frac{\pi(s_e - s_0)}{2 \kappa} \right ), @@ -28,10 +28,9 @@ where: - $\varepsilon_e$ is the element viscosity -- $\varepsilon_0 ~\sim h/p$ is a reference viscosity - $s_e = \log_{10}{S_e} \sim 1/p^4$ is the smoothness indicator - $s_0$ is a reference smoothness value -- $\kappa$ controls the width of the transition between 0 to $\varepsilon_0$ +- $\kappa$ controls the width of the transition between 0 to 1 Smoothness Indicator Evaluation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -103,8 +102,29 @@ def compute_smoothness_indicator(): return knl -def smoothness_indicator(u, discr, kappa=1.0, s0=-6.0): - """Calculate the smoothness indicator.""" +def smoothness_indicator(discr, u, kappa=1.0, s0=-6.0): + """Calculate the smoothness indicator. + + Parameters + ---------- + u + A DOF Array of the field that is used to calculate the + smoothness indicator. + + kappa + A optional argument that sets the controls the width of the + transition between 0 to 1. + s0 + A optional argument that sets the smoothness level to limit + on. Logical values are [0,-infinity) where -infinity results in + all cells being tagged and 0 results in none. + + Returns + ------- + meshmode.dof_array.DOFArray + A DOF Array containing elementwise constant values between 0 and 1 + which indicate the smoothness of a given element. + """ assert isinstance(u, DOFArray) def get_kernel(): From cb8be6a102cb0b7089e724324761b97546480db9 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 15 Mar 2021 15:52:44 -0700 Subject: [PATCH 043/385] Remove linting modification to simutil.py --- mirgecom/simutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 0347ec5db..42700fd0e 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -121,7 +121,7 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, if do_viz: io_fields = [ ("cv", cv), - ("dv", dependent_vars), + ("dv", dependent_vars) ] if exact_soln is not None: exact_list = [ From 0f2d680a005b9175944cc63f8e913ad923d8ae8e Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 15 Mar 2021 16:28:26 -0700 Subject: [PATCH 044/385] Fixed tests to reflect change in smoothness indicator parameters. --- test/test_artificial_viscosity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_artificial_viscosity.py b/test/test_artificial_viscosity.py index 822a7eaa9..ef36074ad 100644 --- a/test/test_artificial_viscosity.py +++ b/test/test_artificial_viscosity.py @@ -61,7 +61,7 @@ def test_tag_cells(ctx_factory, dim, order): tolerance = 1.e-16 def norm_indicator(expected, discr, soln, **kwargs): - return(discr.norm(expected-smoothness_indicator(soln, discr, **kwargs), + return(discr.norm(expected-smoothness_indicator(discr, soln, **kwargs), np.inf)) from meshmode.mesh.generation import generate_regular_rect_mesh From 834a98cd5946510092ed12c29b987b723262e930 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Fri, 26 Mar 2021 12:35:10 -0500 Subject: [PATCH 045/385] Update mirgecom/initializers.py Co-authored-by: Mike Campbell --- mirgecom/initializers.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index f9510c349..f31a20f96 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -350,12 +350,11 @@ def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): + rhoer/2.0*(actx.np.tanh(xtanh)+1.0)) u = ul/2.0*(actx.np.tanh(-xtanh)+1.0)+ur/2.0*(actx.np.tanh(xtanh)+1.0) v = vl/2.0*(actx.np.tanh(-xtanh)+1.0)+vr/2.0*(actx.np.tanh(xtanh)+1.0) - rhou = mass*u - rhov = mass*v - energy = rhoe + 0.5*mass*(u*u + v*v) + vel = make_obj_array([u, v]) + mom = mass * vel + energy = rhoe + .5*mass*np.dot(vel, vel) - return join_conserved(dim=self._dim, mass=mass, energy=energy, - momentum=make_obj_array([rhou, rhov])) + return join_conserved(dim=self._dim, mass=mass, energy=energy, momentum=mom) class Lump: From d72df2abaa2155d7f5942c0e28779baa17f20ae3 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Sun, 25 Apr 2021 14:12:52 -0500 Subject: [PATCH 046/385] Refactor AV (#12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * CI: install mirgecom branch directly (#315) * Run flake8 in Github CI on target Py version (#313) * Run flake8 in Github CI on target Py version * Bump Python compat target to 3.8 * Bump Py compatibility target in Github CI to 3.8 Co-authored-by: Matthias Diener * fix CI runs (#316) - Run `brew update` to fix spurious homebrew downloading errors - Make sure to be able to run without a specified branch (for e.g. scheduled CI runs) - prefer mpich over openmpi, as it can show better error messages in CI * skip clone on CI (#318) * Update half-baked, push to save. * Refactor AV to address remaining review comments, merge with main, make it closer to consistent with NS. * Fix up example to use new AV, remove temporary fix for resolved issue. * Fix slipwall boundary issue for multispecies * Tweak for multispecies. * Add documentation for Persson smoothness indicator * Remove function/module name ambiguity * Update doublemach to use av_operator interface * Update test to use av_operator interface * Update docs to correct rendering, formatting * Update docs to correct rendering, formatting * Update docs to correct rendering, formatting Co-authored-by: Matthias Diener Co-authored-by: Andreas Klöckner --- .ci-support/install.sh | 4 +- .github/workflows/ci.yaml | 6 +- doc/misc.rst | 3 + doc/operators/gas-dynamics.rst | 3 +- examples/doublemach-mpi.py | 24 +- mirgecom/artificial_viscosity.py | 311 +++++++++++++----- mirgecom/boundary.py | 68 ++-- mirgecom/fluid.py | 16 + mirgecom/tag_cells.py | 179 ---------- setup.py | 7 +- ...est_artificial_viscosity.py => test_av.py} | 40 ++- 11 files changed, 323 insertions(+), 338 deletions(-) delete mode 100644 mirgecom/tag_cells.py rename test/{test_artificial_viscosity.py => test_av.py} (86%) diff --git a/.ci-support/install.sh b/.ci-support/install.sh index 1327d875a..6b57e2d34 100644 --- a/.ci-support/install.sh +++ b/.ci-support/install.sh @@ -1,11 +1,13 @@ if [ "$(uname)" = "Darwin" ]; then PLATFORM=MacOSX +brew update +brew upgrade brew install open-mpi brew install octave else PLATFORM=Linux sudo apt-get update -sudo apt-get -y install openmpi-bin libopenmpi-dev +sudo apt-get -y install libmpich-dev mpich sudo apt-get -y install octave fi MINIFORGE_INSTALL_DIR=.miniforge3 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ffd78267f..9e626c4f8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,8 @@ jobs: - uses: actions/setup-python@v1 with: - python-version: '3.x' + # matches compat target in setup.py + python-version: '3.8' - name: Flake8 test run: | python3 -m venv myenv @@ -143,9 +144,8 @@ jobs: cd .. git clone https://github.com/illinois-ceesd/emirge cd emirge - sed -i.bak '/fetch-mirgecom/d' install.sh cp -a ../mirgecom . - ./install.sh + ./install.sh --skip-clone - name: Run simple mirgecom test run: | diff --git a/doc/misc.rst b/doc/misc.rst index b79553f7f..e5423c112 100644 --- a/doc/misc.rst +++ b/doc/misc.rst @@ -63,3 +63,6 @@ References `DOI: `__ .. [Bassi_1997] F. Bassi and S. Rebay (1997), Journal of Computational Physics 131 \ `(DOI) `__ +.. [Persson_2012] P. Persson and J. Peraire, AIAA 44 \ + `(DOI) `__ + diff --git a/doc/operators/gas-dynamics.rst b/doc/operators/gas-dynamics.rst index bcb0336ed..48aa91507 100644 --- a/doc/operators/gas-dynamics.rst +++ b/doc/operators/gas-dynamics.rst @@ -7,4 +7,5 @@ Gas Dynamics .. automodule:: mirgecom.inviscid .. automodule:: mirgecom.flux .. automodule:: mirgecom.boundary -.. automodule:: mirgecom.euler \ No newline at end of file +.. automodule:: mirgecom.euler +.. automodule:: mirgecom.artificial_viscosity diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 2bd1c719e..4b3036799 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -36,8 +36,10 @@ from mirgecom.euler import inviscid_operator, split_conserved -from mirgecom.artificial_viscosity import artificial_viscosity -from mirgecom.tag_cells import smoothness_indicator +from mirgecom.artificial_viscosity import ( + av_operator, + smoothness_indicator +) from mirgecom.simutil import ( inviscid_sim_timestep, sim_checkpoint, @@ -113,7 +115,9 @@ def main(ctx_factory=cl.create_some_context): dim = 2 order = 3 - t_final = 1.0e-2 + # Too many steps for CI + # t_final = 1.0e-2 + t_final = 1.0e-3 current_cfl = 0.1 current_dt = 1.0e-4 current_t = 0 @@ -154,16 +158,6 @@ def main(ctx_factory=cl.create_some_context): discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) - # Temporary fix for Issue #280 - local_boundaries = {} - for btag in boundaries: - bnd_discr = discr.discr_from_dd(btag) - bnd_nodes = thaw(actx, bnd_discr.nodes()) - if bnd_nodes[0][0].shape[0] > 0: - local_boundaries[btag] = boundaries[btag] - boundaries = local_boundaries - # End fix - nodes = thaw(actx, discr.nodes()) current_state = initializer(nodes) @@ -203,8 +197,8 @@ def main(ctx_factory=cl.create_some_context): def my_rhs(t, state): return inviscid_operator( discr, q=state, t=t, boundaries=boundaries, eos=eos - ) + artificial_viscosity( - discr, t=t, r=state, boundaries=boundaries, alpha=alpha, eos=eos, + ) + av_operator( + discr, t=t, q=state, boundaries=boundaries, alpha=alpha, eos=eos, s0=s0, kappa=kappa ) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 7b9f4df0c..290af1201 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -1,30 +1,70 @@ -r""":mod:`mirgecom.artificial viscosity` Artificial viscocity for Euler. +r""":mod:`mirgecom.artificial_viscosity` Artificial viscocity for Euler. -Artificial viscosity for the Euler Equations: +Euler Equations with artificial viscosity term: .. math:: - \partial_t \mathbf{R} = \nabla\cdot{\varepsilon\nabla\mathbf{R}} + \partial_t \mathbf{Q} = -\nabla\cdot\mathbf{F}^I + + \nabla\cdot{\varepsilon\nabla\mathbf{Q}} where: -- state $\mathbf{R} = [\rho, \rho{E}, \rho\vec{V}, \rho{Y}_\alpha]$ -- artifical viscosity coefficient $\varepsilon$ +- fluid state: $\mathbf{Q} = [\rho, \rho{E}, \rho\mathbf{V}, \rho\mathbf{Y}]$ +- inviscid fluxes: $\mathbf{F}^I$ +- artifical viscosity coefficient: $\varepsilon$ To evalutate the second order derivative the problem is recast as a set of first order problems: .. math:: - \partial_t \mathbf{R} = \nabla\cdot{\mathbf{Q}} - \mathbf{Q} = \varepsilon\nabla\mathbf{R} + \partial_t{\mathbf{Q}} &= \nabla\cdot\mathbf{R} -\nabla\cdot\mathbf{F}^I \\ + \mathbf{R} &= \varepsilon\nabla\mathbf{Q} -where $\mathbf{Q}$ is an intermediate variable. +where $\mathbf{R}$ is an intermediate variable, and the artitifial viscosity +coefficient, $\varepsilon$, is spatially dependent and calculated using the +smoothness indicator of [Persson_2012]_: -RHS Evaluation -^^^^^^^^^^^^^^ +.. math:: + + S_e = \frac{\langle u_{N_p} - u_{N_{p-1}}, u_{N_p} - + u_{N_{p-1}}\rangle_e}{\langle u_{N_p}, u_{N_p} \rangle_e} + +where: + +- $S_e$ is the smoothness indicator +- $u_{N_p}$ is the solution in modal basis at the current polynomial order +- $u_{N_{p-1}}$ is the truncated modal represention to the polynomial order $p-1$ +- The $L_2$ inner product on an element is denoted $\langle \cdot,\cdot \rangle_e$ + +The elementwise viscoisty is then calculated: + +.. math:: + + \varepsilon_e = + \begin{cases} + 0, & s_e < s_0 - \kappa \\ + \frac{1}{2}\left( 1 + \sin \frac{\pi(s_e - s_0)}{2 \kappa} \right ), + & s_0-\kappa \le s_e \le s_0+\kappa \\ + 1, & s_e > s_0+\kappa + \end{cases} + +where: + +- $\varepsilon_e$ is the element viscosity +- $s_e = \log_{10}{S_e} \sim 1/p^4$ is the smoothness indicator +- $s_0$ is a reference smoothness value +- $\kappa$ controls the width of the transition between 0 to 1 -.. autofunction:: artificial_viscosity +Smoothness Indicator Evaluation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. autofunction:: smoothness_indicator + +AV RHS Evaluation +^^^^^^^^^^^^^^^^^ + +.. autofunction:: av_operator """ __copyright__ = """ @@ -52,22 +92,27 @@ """ import numpy as np -from pytools.obj_array import ( - obj_array_vectorize, - obj_array_vectorize_n_args, -) -from meshmode.dof_array import thaw +import loopy as lp +from modepy import vandermonde +from pytools.obj_array import obj_array_vectorize +from meshmode.dof_array import thaw, DOFArray from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.eager import interior_trace_pair, cross_rank_trace_pairs from grudge.symbolic.primitives import TracePair -from mirgecom.tag_cells import smoothness_indicator +from mirgecom.fluid import ( + split_conserved, + join_conserved_vectors +) -def _facial_flux_r(discr, q_tpair): +def _facial_flux_q(discr, q_tpair): - actx = q_tpair.int.array_context + q_int = q_tpair.int + actx = q_int[0].array_context flux_dis = q_tpair.avg + if isinstance(flux_dis, np.ndarray): + flux_dis = flux_dis.reshape(-1, 1) normal = thaw(actx, discr.normal(q_tpair.dd)) @@ -76,32 +121,29 @@ def _facial_flux_r(discr, q_tpair): return discr.project(q_tpair.dd, "all_faces", flux_out) -def _facial_flux_q(discr, q_tpair): +def _facial_flux_r(discr, r_tpair): + r_int = r_tpair.int + actx = r_int[0][0].array_context - actx = q_tpair[0].int.array_context + normal = thaw(actx, discr.normal(r_tpair.dd)) - normal = thaw(actx, discr.normal(q_tpair.dd)) + flux_out = r_tpair.avg @ normal - flux_out = np.dot(q_tpair.avg, normal) + return discr.project(r_tpair.dd, "all_faces", flux_out) - return discr.project(q_tpair.dd, "all_faces", flux_out) - -def artificial_viscosity(discr, t, eos, boundaries, r, alpha, **kwargs): +def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): r"""Compute artifical viscosity for the euler equations. - Calculates - ---------- - numpy.ndarray - The right-hand-side for artificial viscosity for the euler equations. + Computes the the right-hand-side term for artificial viscosity. - .. math:: + .. math:: - \dot{\nabla\cdot{\varepsilon\nabla\mathbf{R}}} + \mbox{RHS}_{\mbox{av}} = \nabla\cdot{\varepsilon\nabla\mathbf{Q}} Parameters ---------- - r + q State array which expects the quantity to be limited on to be listed first in the array. For the Euler equations this could be the canonical conserved variables (mass, energy, mometum) for the fluid along with a @@ -125,72 +167,165 @@ def artificial_viscosity(discr, t, eos, boundaries, r, alpha, **kwargs): Agglomerated object array of DOF Arrays representing the RHS associated with the artificial viscosity application. """ - # Get smoothness indicator - indicator = smoothness_indicator(discr, r[0], **kwargs) - - dflux_r = obj_array_vectorize(discr.weak_grad, r) - - def my_facialflux_r_interior(q): - qin = interior_trace_pair(discr, q) - return _facial_flux_r(discr, q_tpair=qin) + dim = discr.dim + cv = split_conserved(dim, q) - iff_r = obj_array_vectorize(my_facialflux_r_interior, r) + # Get smoothness indicator based on fluid mass density + indicator = smoothness_indicator(discr, cv.mass, **kwargs) - def my_facialflux_r_partition(q): - qin = cross_rank_trace_pairs(discr, q) - return sum(_facial_flux_r(discr, q_tpair=part_pair) for part_pair in qin) + grad_q_vol = join_conserved_vectors(dim, obj_array_vectorize(discr.weak_grad, q)) - pbf_r = obj_array_vectorize(my_facialflux_r_partition, r) - - dbf_r = np.zeros_like(iff_r) + q_flux_int = _facial_flux_q(discr, q_tpair=interior_trace_pair(discr, q)) + q_flux_pb = sum(_facial_flux_q(discr, q_tpair=pb_tpair) + for pb_tpair in cross_rank_trace_pairs(discr, q)) + q_flux_db = 0 for btag in boundaries: - - def my_facialflux_r_boundary(sol_ext, sol_int): - q_tpair = TracePair( - btag, - interior=sol_int, - exterior=sol_ext, - ) - return _facial_flux_r(discr, q_tpair=q_tpair) - - r_ext = boundaries[btag].exterior_soln(discr, btag=btag, t=t, q=r, eos=eos) - r_int = discr.project("vol", btag, r) - dbf_r = dbf_r + obj_array_vectorize_n_args( - my_facialflux_r_boundary, r_ext, r_int - ) - - # Compute q, half way done! - q = discr.inverse_mass( - -alpha * indicator * (dflux_r - discr.face_mass(iff_r + pbf_r + dbf_r)) + q_tpair = TracePair( + btag, + interior=discr.project("vol", btag, q), + exterior=boundaries[btag].exterior_q(discr, btag=btag, t=t, + q=q, eos=eos)) + q_flux_db = q_flux_db + _facial_flux_q(discr, q_tpair=q_tpair) + q_bnd_flux = q_flux_int + q_flux_pb + q_flux_db + + # Compute R + r = discr.inverse_mass( + -alpha * indicator * (grad_q_vol - discr.face_mass(q_bnd_flux)) ) - dflux_q = obj_array_vectorize(discr.weak_div, q) - - def my_facialflux_q_interior(q): - qin = interior_trace_pair(discr, q) - iff_q = _facial_flux_q(discr, q_tpair=qin) - return iff_q + div_r_vol = discr.weak_div(r) + r_flux_int = _facial_flux_r(discr, r_tpair=interior_trace_pair(discr, r)) + r_flux_pb = sum(_facial_flux_r(discr, r_tpair=pb_tpair) + for pb_tpair in cross_rank_trace_pairs(discr, r)) + r_flux_db = 0 + for btag in boundaries: + r_tpair = TracePair( + btag, + interior=discr.project("vol", btag, r), + exterior=boundaries[btag].exterior_grad_q(discr, btag=btag, t=t, + grad_q=r, eos=eos)) + r_flux_db = r_flux_db + _facial_flux_r(discr, r_tpair=r_tpair) + r_flux_bnd = r_flux_int + r_flux_pb + r_flux_db + + # Return the AV RHS term + return discr.inverse_mass(-div_r_vol + discr.face_mass(r_flux_bnd)) + + +def artificial_viscosity(discr, t, eos, boundaries, q, alpha, **kwargs): + """Interface :function:`av_operator` with backwards-compatible API.""" + from warnings import warn + warn("Do not call artificial_viscosity; it is now called av_operator. This" + "function will disappear in 2021", DeprecationWarning, stacklevel=2) + return av_operator(discr=discr, eos=eos, boundaries=boundaries, + q=q, alpha=alpha, t=t) + + +def linear_operator_kernel(): + """Apply linear operator to all elements.""" + from meshmode.array_context import make_loopy_program + + knl = make_loopy_program( + """{[iel,idof,j]: + 0<=iel (s0 - kappa) + yesnou = indicator > (s0 + kappa) + sin_indicator = actx.np.where( + yesnol, + 0.5 * (1.0 + actx.np.sin(np.pi * (indicator - s0) / (2.0 * kappa))), + 0.0 * indicator, + ) + indicator = actx.np.where(yesnou, 1.0 + 0.0 * indicator, sin_indicator) - # Return the rhs contribution - return discr.inverse_mass(-dflux_q + discr.face_mass(iff_q + pbf_q + dbf_q)) + return indicator diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 1bc751354..65be0ed17 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -37,7 +37,11 @@ from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa # from mirgecom.eos import IdealSingleGas from grudge.symbolic.primitives import TracePair -from mirgecom.fluid import split_conserved, join_conserved +from mirgecom.fluid import ( + split_conserved, + join_conserved, + join_conserved_vectors +) class PrescribedBoundary: @@ -62,11 +66,11 @@ def __init__(self, userfunc): def boundary_pair(self, discr, q, btag, **kwargs): """Get the interior and exterior solution on the boundary.""" - ext_soln = self.exterior_soln(discr, q, btag, **kwargs) + ext_soln = self.exterior_q(discr, q, btag, **kwargs) int_soln = discr.project("vol", btag, q) return TracePair(btag, interior=int_soln, exterior=ext_soln) - def exterior_soln(self, discr, q, btag, **kwargs): + def exterior_q(self, discr, q, btag, **kwargs): """Get the exterior solution on the boundary.""" actx = q[0].array_context @@ -75,9 +79,9 @@ def exterior_soln(self, discr, q, btag, **kwargs): ext_soln = self._userfunc(nodes, **kwargs) return ext_soln - def av(self, discr, q, btag, **kwargs): + def exterior_grad_q(self, discr, grad_q, btag, **kwargs): """Get the exterior solution on the boundary.""" - return discr.project("vol", btag, q) + return discr.project("vol", btag, grad_q) class DummyBoundary: @@ -88,17 +92,17 @@ class DummyBoundary: def boundary_pair(self, discr, q, btag, **kwargs): """Get the interior and exterior solution on the boundary.""" - dir_soln = self.exterior_soln(discr, q, btag, **kwargs) + dir_soln = self.exterior_q(discr, q, btag, **kwargs) return TracePair(btag, interior=dir_soln, exterior=dir_soln) - def exterior_soln(self, discr, q, btag, **kwargs): + def exterior_q(self, discr, q, btag, **kwargs): """Get the exterior solution on the boundary.""" dir_soln = discr.project("vol", btag, q) return dir_soln - def av(self, discr, q, btag, **kwargs): - """Get the exterior solution on the boundary.""" - return discr.project("vol", btag, q) + def exterior_grad_q(self, discr, grad_q, btag, **kwargs): + """Get the grad_q on the exterior of the boundary.""" + return discr.project("vol", btag, grad_q) class AdiabaticSlipBoundary: @@ -121,12 +125,12 @@ class AdiabaticSlipBoundary: def boundary_pair(self, discr, q, btag, **kwargs): """Get the interior and exterior solution on the boundary.""" - bndry_soln = self.exterior_soln(discr, q, btag, **kwargs) + bndry_soln = self.exterior_q(discr, q, btag, **kwargs) int_soln = discr.project("vol", btag, q) return TracePair(btag, interior=int_soln, exterior=bndry_soln) - def exterior_soln(self, discr, q, btag, **kwargs): + def exterior_q(self, discr, q, btag, **kwargs): """Get the exterior solution on the boundary. The exterior solution is set such that there will be vanishing @@ -160,40 +164,42 @@ def exterior_soln(self, discr, q, btag, **kwargs): # Form the external boundary solution with the new momentum bndry_soln = join_conserved(dim=dim, mass=int_cv.mass, energy=int_cv.energy, - momentum=ext_mom) + momentum=ext_mom, + species_mass=int_cv.species_mass) return bndry_soln - def av(self, discr, q, btag, **kwargs): - """Get the exterior solution on the boundary.""" + def exterior_grad_q(self, discr, grad_q, btag, **kwargs): + """Get the exterior grad(Q) on the boundary.""" # Grab some boundary-relevant data - dim = discr.dim - cv = split_conserved(dim, q) + num_equations, dim = grad_q.shape + num_species = num_equations - dim - 2 + cv = split_conserved(dim, grad_q) actx = cv.mass[0].array_context - # Grab a unit normal to the boundary normal = thaw(actx, discr.normal(btag)) # Get the interior soln - int_soln = discr.project("vol", btag, q) - bndry_q = split_conserved(dim, int_soln) + int_soln = discr.project("vol", btag, grad_q) + int_cv = split_conserved(dim, int_soln) # create result array to fill - result = np.zeros(2+dim, dtype=object) + result = np.empty(shape=grad_q.shape, dtype=object) # flip signs on mass and energy # to apply a neumann condition on q - result[0] = -1.0*bndry_q.mass - result[1] = -1.0*bndry_q.energy + result[0] = -int_cv.mass + result[1] = -int_cv.energy # Subtract 2*wall-normal component of q # to enforce q=0 on the wall # flip remaining components to set a neumann condition - from pytools.obj_array import make_obj_array - q_mom_normcomp = make_obj_array( - [np.outer(normal, np.dot(bndry_q.momentum, normal))[i] - for i in range(dim)] - ) - result[2:] = -1*(bndry_q.momentum-2.0*q_mom_normcomp) - - return(result) + s_mom_normcomp = np.outer(normal, np.dot(int_cv.momentum, normal)) + s_mom_flux = 2*s_mom_normcomp - int_cv.momentum + for idim in range(dim): + result[2+idim] = s_mom_flux[idim] + + for ispec in range(num_species): + result[dim+2+ispec] = -int_cv.species_mass[ispec] + + return join_conserved_vectors(dim, result) diff --git a/mirgecom/fluid.py b/mirgecom/fluid.py index a83ac13f5..33e3dac3b 100644 --- a/mirgecom/fluid.py +++ b/mirgecom/fluid.py @@ -6,6 +6,7 @@ .. autoclass:: ConservedVars .. autofunction:: split_conserved .. autofunction:: join_conserved +.. autofunction:: join_conserved_vectors Helper Functions ^^^^^^^^^^^^^^^^ @@ -165,6 +166,21 @@ def join_conserved(dim, mass, energy, momentum, return result +def join_conserved_vectors(dim, ary): + r"""Create a 2D array of shape(len(ary), dim).""" + neq = len(ary) + nspecies = neq - (dim+2) + retval = np.empty(shape=(neq, dim), dtype=object) + cv = split_conserved(dim, ary) + retval[0] = cv.mass + retval[1] = cv.energy + for i in range(dim): + retval[2+i] = ary[2+i] + for i in range(nspecies): + retval[dim+2+i] = ary[dim+2+i] + return retval + + def velocity_gradient(discr, cv, grad_cv): r""" Compute the gradient of fluid velocity. diff --git a/mirgecom/tag_cells.py b/mirgecom/tag_cells.py deleted file mode 100644 index 539cbf97c..000000000 --- a/mirgecom/tag_cells.py +++ /dev/null @@ -1,179 +0,0 @@ -r""":mod:`mirgecom.tag_cells` Compute smoothness indicator. - -Evalutes the smoothness indicator of Persson: - -.. math:: - - S_e = \frac{\langle u_{N_p} - u_{N_{p-1}}, u_{N_p} - - u_{N_{p-1}}\rangle_e}{\langle u_{N_p}, u_{N_p} \rangle_e} - -where: -- $S_e$ is the smoothness indicator -- $u_{N_p}$ is the modal representation of the solution at the current polynomial - order -- $u_{N_{p-1}}$ is the truncated modal represention to the polynomial order $p-1$ -- The $L_2$ inner product on an element is denoted $\langle \cdot,\cdot \rangle_e$ - -The elementwise viscoisty is then calculated: - -.. math:: - - \varepsilon_e = - \begin{cases} - 0, & s_e < s_0 - \kappa \\ - \frac{1}{2}\left( 1 + \sin \frac{\pi(s_e - s_0)}{2 \kappa} \right ), - & s_0-\kappa \le s_e \le s_0+\kappa \\ - 1, & s_e > s_0+\kappa - \end{cases} - -where: -- $\varepsilon_e$ is the element viscosity -- $s_e = \log_{10}{S_e} \sim 1/p^4$ is the smoothness indicator -- $s_0$ is a reference smoothness value -- $\kappa$ controls the width of the transition between 0 to 1 - -Smoothness Indicator Evaluation -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. autofunction:: smoothness_indicator -""" - -__copyright__ = """ -Copyright (C) 2020 University of Illinois Board of Trustees -""" - -__license__ = """ -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -""" - -import numpy as np -import loopy as lp -from meshmode.dof_array import DOFArray -from modepy import vandermonde - - -def linear_operator_kernel(): - """Apply linear operator to all elements.""" - from meshmode.array_context import make_loopy_program - - knl = make_loopy_program( - """{[iel,idof,j]: - 0<=iel (s0 - kappa) - yesnou = indicator > (s0 + kappa) - sin_indicator = actx.np.where( - yesnol, - 0.5 * (1.0 + actx.np.sin(np.pi * (indicator - s0) / (2.0 * kappa))), - 0.0 * indicator, - ) - indicator = actx.np.where(yesnou, 1.0 + 0.0 * indicator, sin_indicator) - - return indicator diff --git a/setup.py b/setup.py index ea49258d4..5a2e5a4f9 100644 --- a/setup.py +++ b/setup.py @@ -27,10 +27,7 @@ def main(): "License :: OSI Approved :: MIT License", "Natural Language :: English", "Programming Language :: Python", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Information Analysis", "Topic :: Scientific/Engineering :: Mathematics", @@ -41,7 +38,7 @@ def main(): packages=find_packages(), - python_requires="~=3.6", + python_requires="~=3.8", install_requires=[ "mpi4py>=3", diff --git a/test/test_artificial_viscosity.py b/test/test_av.py similarity index 86% rename from test/test_artificial_viscosity.py rename to test/test_av.py index ef36074ad..fa6a7e9e3 100644 --- a/test/test_artificial_viscosity.py +++ b/test/test_av.py @@ -29,15 +29,19 @@ import numpy as np import pyopencl as cl import pytest +from pytools.obj_array import make_obj_array from meshmode.array_context import PyOpenCLArrayContext from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL - -from mirgecom.tag_cells import smoothness_indicator -from mirgecom.artificial_viscosity import artificial_viscosity +from mirgecom.fluid import ( + join_conserved, +) +from mirgecom.artificial_viscosity import ( + av_operator, + smoothness_indicator +) from mirgecom.boundary import DummyBoundary from grudge.eager import EagerDGDiscretization -from pytools.obj_array import flat_obj_array from pyopencl.tools import ( # noqa pytest_generate_tests_for_pyopencl as pytest_generate_tests, ) @@ -179,22 +183,28 @@ def test_artificial_viscosity(ctx_factory, dim, order): boundaries = {BTAG_ALL: DummyBoundary()} # Uniform field return 0 rhs - fields = flat_obj_array(zeros+1.0) - rhs = artificial_viscosity(discr, t=0, eos=None, boundaries=boundaries, - r=fields, alpha=1.0, s0=-np.inf) + mass = zeros + 1.0 + energy = zeros + 1.0 + momentum = make_obj_array([zeros + 0 for _ in range(dim)]) + q = join_conserved(dim, mass=mass, energy=energy, momentum=momentum) + rhs = av_operator(discr, t=0, eos=None, boundaries=boundaries, + q=q, alpha=1.0, s0=-np.inf) err = discr.norm(rhs, np.inf) assert err < tolerance - # Linar field return 0 rhs - fields = flat_obj_array(nodes[0]) - rhs = artificial_viscosity(discr, t=0, eos=None, boundaries=boundaries, - r=fields, alpha=1.0, s0=-np.inf) + # Linear field return 0 rhs + mass = nodes[0] + energy = 2.5 + zeros + q = join_conserved(dim, mass=mass, energy=energy, momentum=momentum) + rhs = av_operator(discr, t=0, eos=None, boundaries=boundaries, + q=q, alpha=1.0, s0=-np.inf) err = discr.norm(rhs, np.inf) assert err < tolerance # Quadratic field return constant 2 - fields = flat_obj_array(np.dot(nodes, nodes)) - rhs = artificial_viscosity(discr, t=0, eos=None, boundaries=boundaries, - r=fields, alpha=1.0, s0=-np.inf) - err = discr.norm(2.*dim-rhs, np.inf) + mass = np.dot(nodes, nodes) + q = join_conserved(dim, mass=mass, energy=energy, momentum=momentum) + rhs = av_operator(discr, t=0, eos=None, boundaries=boundaries, + q=q, alpha=1.0, s0=-np.inf) + err = discr.norm(2.*dim-rhs[0], np.inf) assert err < tolerance From 8fc97716acaa7c21d0a7147b56a706d22d03417d Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Mon, 26 Apr 2021 11:50:16 -0500 Subject: [PATCH 047/385] Add some comments for docs, try to address @thomasgibson comments (#13) --- mirgecom/artificial_viscosity.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 290af1201..8cd872b64 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -106,7 +106,7 @@ def _facial_flux_q(discr, q_tpair): - + """Compute facial flux for each scalar component of Q.""" q_int = q_tpair.int actx = q_int[0].array_context @@ -116,17 +116,22 @@ def _facial_flux_q(discr, q_tpair): normal = thaw(actx, discr.normal(q_tpair.dd)) + # This uses a central scalar flux along nhat: + # flux = 1/2 * (Q- + Q+) * nhat flux_out = flux_dis * normal return discr.project(q_tpair.dd, "all_faces", flux_out) def _facial_flux_r(discr, r_tpair): + """Compute facial flux for vector compontnent of grad(Q).""" r_int = r_tpair.int actx = r_int[0][0].array_context normal = thaw(actx, discr.normal(r_tpair.dd)) + # This uses a central vector flux along nhat: + # flux = 1/2 * (grad(Q)- + grad(Q)+) .dot. nhat flux_out = r_tpair.avg @ normal return discr.project(r_tpair.dd, "all_faces", flux_out) @@ -173,11 +178,15 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): # Get smoothness indicator based on fluid mass density indicator = smoothness_indicator(discr, cv.mass, **kwargs) + # R=Grad(Q) volume part grad_q_vol = join_conserved_vectors(dim, obj_array_vectorize(discr.weak_grad, q)) + # R=Grad(Q) Q flux over interior faces q_flux_int = _facial_flux_q(discr, q_tpair=interior_trace_pair(discr, q)) + # R=Grad(Q) Q flux interior faces on partition boundaries q_flux_pb = sum(_facial_flux_q(discr, q_tpair=pb_tpair) for pb_tpair in cross_rank_trace_pairs(discr, q)) + # R=Grad(Q) Q flux domain boundary part (i.e. BCs) q_flux_db = 0 for btag in boundaries: q_tpair = TracePair( @@ -186,6 +195,7 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): exterior=boundaries[btag].exterior_q(discr, btag=btag, t=t, q=q, eos=eos)) q_flux_db = q_flux_db + _facial_flux_q(discr, q_tpair=q_tpair) + # Total Q flux across element boundaries q_bnd_flux = q_flux_int + q_flux_pb + q_flux_db # Compute R @@ -193,10 +203,14 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): -alpha * indicator * (grad_q_vol - discr.face_mass(q_bnd_flux)) ) + # RHS_av = div(R) volume part div_r_vol = discr.weak_div(r) + # RHS_av = div(R): grad(Q) flux interior faces part r_flux_int = _facial_flux_r(discr, r_tpair=interior_trace_pair(discr, r)) + # RHS_av = div(R): grad(Q) flux interior faces on the partition boundaries r_flux_pb = sum(_facial_flux_r(discr, r_tpair=pb_tpair) for pb_tpair in cross_rank_trace_pairs(discr, r)) + # RHS_av = div(R): grad(Q) flux domain boundary part (BCs) r_flux_db = 0 for btag in boundaries: r_tpair = TracePair( @@ -205,6 +219,7 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): exterior=boundaries[btag].exterior_grad_q(discr, btag=btag, t=t, grad_q=r, eos=eos)) r_flux_db = r_flux_db + _facial_flux_r(discr, r_tpair=r_tpair) + # Total grad(Q) flux element boundaries r_flux_bnd = r_flux_int + r_flux_pb + r_flux_db # Return the AV RHS term From 0495f3bd3f2008fcb438c4ec71a3af098beb8d87 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 15:53:20 -0500 Subject: [PATCH 048/385] Update examples/doublemach-mpi.py to get rank properly Co-authored-by: Matt Smith --- examples/doublemach-mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 4b3036799..588bcaaa7 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -147,7 +147,7 @@ def main(ctx_factory=cl.create_some_context): from mpi4py import MPI comm = MPI.COMM_WORLD - rank = comm.Get_rank + rank = comm.Get_rank() gen_grid = partial(get_doublemach_mesh) From b93411b354bda32a3db9c73465aca1d8856a4567 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 15:53:52 -0500 Subject: [PATCH 049/385] Update logic to check if file exists Co-authored-by: Matt Smith --- examples/doublemach-mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 588bcaaa7..7d336be96 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -74,7 +74,7 @@ def get_doublemach_mesh(): ) import os meshfile = "doubleMach.msh" - if os.path.exists(meshfile) is False: + if not os.path.exists(meshfile): mesh = generate_gmsh( ScriptSource(""" x0=1.0/6.0; From 3f336fd76125e6b4a6be8c0cea0c1417413054f4 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 15:54:18 -0500 Subject: [PATCH 050/385] Update documenatation Co-authored-by: Matt Smith --- mirgecom/artificial_viscosity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 8cd872b64..8134a0a53 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -124,7 +124,7 @@ def _facial_flux_q(discr, q_tpair): def _facial_flux_r(discr, r_tpair): - """Compute facial flux for vector compontnent of grad(Q).""" + """Compute facial flux for vector component of grad(Q).""" r_int = r_tpair.int actx = r_int[0][0].array_context From 8fa696ec694dbbd6e5a8c6b24acaf2ac3fda5e49 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 15:54:53 -0500 Subject: [PATCH 051/385] Update documentation Co-authored-by: Matt Smith --- mirgecom/artificial_viscosity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 8134a0a53..213aa1450 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -138,7 +138,7 @@ def _facial_flux_r(discr, r_tpair): def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): - r"""Compute artifical viscosity for the euler equations. + r"""Compute artificial viscosity for the Euler equations. Computes the the right-hand-side term for artificial viscosity. From de305d91ebf81cfe85dbf1f73ac527fd53766e4b Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 15:55:18 -0500 Subject: [PATCH 052/385] Update documentation Co-authored-by: Matt Smith --- mirgecom/artificial_viscosity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 213aa1450..a799821d1 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -155,7 +155,7 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): vector of species masses for multi-component fluids. boundaries - Dicitionary of boundary functions, one for each valid boundary tag + Dictionary of boundary functions, one for each valid boundary tag t Time From 376ba33f2e55b33057740b5f47a78f333c8e79b2 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 15:55:45 -0500 Subject: [PATCH 053/385] Update documentation Co-authored-by: Matt Smith --- mirgecom/artificial_viscosity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index a799821d1..466995aca 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -161,7 +161,7 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): Time alpha - The maximum artifical viscosity coeffiecent to be applied + The maximum artificial viscosity coefficient to be applied eos: mirgecom.eos.GasEOS Only used as a pass through to the boundary conditions. From 184e2cc1728516a200e4513e5a32c3f14fb2e5b3 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 15:57:11 -0500 Subject: [PATCH 054/385] Update documentation Co-authored-by: Matt Smith --- mirgecom/artificial_viscosity.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 466995aca..15ea7ac10 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -169,8 +169,7 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): Returns ------- numpy.ndarray - Agglomerated object array of DOF Arrays representing the RHS associated - with the artificial viscosity application. + The artificial viscosity operator applied to *q*. """ dim = discr.dim cv = split_conserved(dim, q) From 1fea6b686c7ec1c2fc5b23cbe3b8d759cae0c75d Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:00:08 -0500 Subject: [PATCH 055/385] Update documentation Co-authored-by: Matt Smith --- mirgecom/artificial_viscosity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 15ea7ac10..c9584b853 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -281,7 +281,7 @@ def smoothness_indicator(discr, u, kappa=1.0, s0=-6.0): A optional argument that sets the controls the width of the transition between 0 to 1. s0 - A optional argument that sets the smoothness level to limit + An optional argument that sets the smoothness level to limit on. Logical values are [0,-infinity) where -infinity results in all cells being tagged and 0 results in none. From f17e4508105631f245f3d99f694ceaf86132b087 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:00:34 -0500 Subject: [PATCH 056/385] Update documentation Co-authored-by: Matt Smith --- mirgecom/artificial_viscosity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index c9584b853..1f535bfef 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -282,7 +282,7 @@ def smoothness_indicator(discr, u, kappa=1.0, s0=-6.0): transition between 0 to 1. s0 An optional argument that sets the smoothness level to limit - on. Logical values are [0,-infinity) where -infinity results in + on. Values in the range $(-\infty,0]$ are allowed, where $-\infty$ results in all cells being tagged and 0 results in none. Returns From 9cd58d348c22e260be62460e431aab33e10fa167 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:00:56 -0500 Subject: [PATCH 057/385] Update documentation Co-authored-by: Matt Smith --- mirgecom/artificial_viscosity.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 1f535bfef..11143e3c1 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -273,9 +273,8 @@ def smoothness_indicator(discr, u, kappa=1.0, s0=-6.0): Parameters ---------- - u - A DOF Array of the field that is used to calculate the - smoothness indicator. + u: meshmode.dof_array.DOFArray + The field that is used to calculate the smoothness indicator. kappa A optional argument that sets the controls the width of the From cc2752a0e87692b5797d82c713e5de1daf0f6f87 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:01:18 -0500 Subject: [PATCH 058/385] Update documentation Co-authored-by: Matt Smith --- mirgecom/artificial_viscosity.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 11143e3c1..217177229 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -277,8 +277,7 @@ def smoothness_indicator(discr, u, kappa=1.0, s0=-6.0): The field that is used to calculate the smoothness indicator. kappa - A optional argument that sets the controls the width of the - transition between 0 to 1. + An optional argument that controls the width of the transition from 0 to 1. s0 An optional argument that sets the smoothness level to limit on. Values in the range $(-\infty,0]$ are allowed, where $-\infty$ results in From d78cdabd41ea178b2067243d5d53d746b9991e19 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:02:19 -0500 Subject: [PATCH 059/385] Update documentation Co-authored-by: Matt Smith --- mirgecom/artificial_viscosity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 217177229..24ec14b22 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -286,8 +286,8 @@ def smoothness_indicator(discr, u, kappa=1.0, s0=-6.0): Returns ------- meshmode.dof_array.DOFArray - A DOF Array containing elementwise constant values between 0 and 1 - which indicate the smoothness of a given element. + The elementwise constant values between 0 and 1 which indicate the smoothness + of a given element. """ assert isinstance(u, DOFArray) From d55666c2a1a84adc0b36a6ab6e5d215e592edfc6 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:03:15 -0500 Subject: [PATCH 060/385] Update documentation Co-authored-by: Matt Smith --- mirgecom/initializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index e6bbc2370..00cb6a1af 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -268,7 +268,7 @@ class DoubleMachReflection: - Woodward and Collela - The inital condition is defined + The initial condition is defined .. math:: From 06d171c2c74b49e0f39db59dd8b9876d5befaa44 Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 20:16:36 -0700 Subject: [PATCH 061/385] Fix flake8/docstyle issue --- mirgecom/artificial_viscosity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 24ec14b22..35db68006 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -269,7 +269,7 @@ def compute_smoothness_indicator(): def smoothness_indicator(discr, u, kappa=1.0, s0=-6.0): - """Calculate the smoothness indicator. + r"""Calculate the smoothness indicator. Parameters ---------- From 974bcd47752630f73c6b9a536352b4bc128a5b2c Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 26 Apr 2021 20:25:23 -0700 Subject: [PATCH 062/385] Clean up doublemach case documentation and remove dim as parameter. --- examples/doublemach-mpi.py | 2 +- mirgecom/initializers.py | 48 ++++++++++++++++++++++++++++---------- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 7d336be96..17d538466 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -122,7 +122,7 @@ def main(ctx_factory=cl.create_some_context): current_dt = 1.0e-4 current_t = 0 eos = IdealSingleGas() - initializer = DoubleMachReflection(dim) + initializer = DoubleMachReflection() casename = "doubleMach" from grudge import sym diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 00cb6a1af..35d001627 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -272,46 +272,70 @@ class DoubleMachReflection: .. math:: - (\rho,u,v,P) = + {\rho}(x < x_s(y,t)) &= \gamma \rho_j\\ + {\rho}(x > x_s(y,t)) &= \gamma\\ + {\rho}{V_x}(x < x_s(y,t)) &= u_p \cos(\pi/6)\\ + {\rho}{V_x}(x > x_s(y,t)) &= 0\\ + {\rho}{V_y}(x > x_s(y,t)) &= u_p \sin(\pi/6)\\ + {\rho}{V_y}(x > x_s(y,t)) &= 0\\ + {\rho}E(x < x_s(y,t)) &= (\gamma-1)p_j\\ + {\rho}E(x > x_s(y,t)) &= (\gamma-1) - This function only serves as an initial condition + where the shock position is given, + + .. math:: + + x_s = x_0 + y/\sqrt{3} + 2 u_s t/\sqrt{3} + + and the normal shock jump relations are + + .. math:: + + \rho_j &= \frac{(\gamma + 1) u_s^2}{(\gamma-1) u_s^2 + 2} \\ + p_j &= \frac{2 \gamma u_s^2 - (\gamma - 1)}{\gamma+1} \\ + u_p &= 2 \frac{u_s^2-1}{(\gamma+1) u_s} + + The initial shock location is given by $x_0$ and $u_s$ is the shock speed. + This function serves as an initial condition as well as boundary conditions + for $t>0$ .. automethod:: __init__ .. automethod:: __call__ """ def __init__( - self, dim=2, shock_location=1.0/6.0, shock_speed=4.0 + self, shock_location=1.0/6.0, shock_speed=4.0 ): """Initialize initial condition options. Parameters ---------- - dim: int - dimension of domain, must be 2 shock_location: float - location of shock + initial location of shock shock_speed: float shock speed, Mach number """ self._shock_location = shock_location - self._dim = dim self._shock_speed = shock_speed def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): - """ - Create the 1D Sod's shock solution at locations *x_vec*. + r""" + Create the initial condition for the double Mach reflection case at + locations *x_vec*. Also provides boundary conditions for solution at + times $t>0$. Parameters ---------- t: float - Current time at which the solution is desired (unused) + Current time of solution when called as a boundary condition x_vec: numpy.ndarray Nodal coordinates eos: :class:`mirgecom.eos.GasEOS` Equation of state class to be used in construction of soln (if needed) """ - assert self._dim == 2, "only defined for dim=2" + # Fail if numdim is other than 2 + if(len(x_vec)) != 2: + raise ValueError("Case only defined for 2 dimensions") gm1 = eos.gamma() - 1.0 gp1 = eos.gamma() + 1.0 @@ -354,7 +378,7 @@ def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): mom = mass * vel energy = rhoe + .5*mass*np.dot(vel, vel) - return join_conserved(dim=self._dim, mass=mass, energy=energy, momentum=mom) + return join_conserved(dim=2, mass=mass, energy=energy, momentum=mom) class Lump: From f3e6a6110968876e40be153abdb8fb5787b32ecd Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 27 Apr 2021 11:12:19 -0500 Subject: [PATCH 063/385] Tweak the doublemach docs and implementation just a bit (#15) * Tweak the doublemach docs and implementation just a bit to remove some unneeded stuff. * tweak verbiage per @w-hagen suggestion to be more correct and complete --- doc/misc.rst | 3 ++- mirgecom/initializers.py | 52 +++++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/doc/misc.rst b/doc/misc.rst index e5423c112..b9bf6a445 100644 --- a/doc/misc.rst +++ b/doc/misc.rst @@ -65,4 +65,5 @@ References `(DOI) `__ .. [Persson_2012] P. Persson and J. Peraire, AIAA 44 \ `(DOI) `__ - +.. [Woodward_1984] Woodward and Colella, Journal of Computational Physics, 54 \ + `(DOI) `__ diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 35d001627..0caa5c6c5 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -266,9 +266,8 @@ def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): class DoubleMachReflection: r"""Implement the double shock reflection problem. - - Woodward and Collela - - The initial condition is defined + The double shock reflection solution is crafted after [Woodward_1984]_ + and is defined by: .. math:: @@ -279,7 +278,7 @@ class DoubleMachReflection: {\rho}{V_y}(x > x_s(y,t)) &= u_p \sin(\pi/6)\\ {\rho}{V_y}(x > x_s(y,t)) &= 0\\ {\rho}E(x < x_s(y,t)) &= (\gamma-1)p_j\\ - {\rho}E(x > x_s(y,t)) &= (\gamma-1) + {\rho}E(x > x_s(y,t)) &= (\gamma-1), where the shock position is given, @@ -296,8 +295,6 @@ class DoubleMachReflection: u_p &= 2 \frac{u_s^2-1}{(\gamma+1) u_s} The initial shock location is given by $x_0$ and $u_s$ is the shock speed. - This function serves as an initial condition as well as boundary conditions - for $t>0$ .. automethod:: __init__ .. automethod:: __call__ @@ -306,7 +303,7 @@ class DoubleMachReflection: def __init__( self, shock_location=1.0/6.0, shock_speed=4.0 ): - """Initialize initial condition options. + """Initialize double shock reflection parameters. Parameters ---------- @@ -320,14 +317,18 @@ def __init__( def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): r""" - Create the initial condition for the double Mach reflection case at - locations *x_vec*. Also provides boundary conditions for solution at - times $t>0$. + Create double mach reflection solution at locations *x_vec*. + + At times $t > 0$, calls to this routine create an advanced solution + under the assumption of constant normal shock speed *shock_speed*. + The advanced solution *is not* the exact solution, but is appropriate + for use as an exact boundary solution on the top and upstream (left) + side of the domain. Parameters ---------- t: float - Current time of solution when called as a boundary condition + Time at which to compute the solution x_vec: numpy.ndarray Nodal coordinates eos: :class:`mirgecom.eos.GasEOS` @@ -344,29 +345,23 @@ def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): y_rel = x_vec[1] actx = x_rel.array_context - zeros = 0*x_rel - - shock_location = zeros + self._shock_location - shock_speed = zeros + self._shock_speed - t = zeros + t - # Normal Shock Relations shock_speed_2 = self._shock_speed * self._shock_speed rho_jump = gp1 * shock_speed_2 / (gm1 * shock_speed_2 + 2.) p_jump = (2. * eos.gamma() * shock_speed_2 - gm1) / gp1 up = 2. * (shock_speed_2 - 1.) / (gp1 * self._shock_speed) - rhol = zeros + eos.gamma() * rho_jump - rhor = zeros + eos.gamma() - ul = zeros + up * np.cos(np.pi/6.0) - ur = zeros + 0.0 - vl = zeros - up * np.sin(np.pi/6.0) - vr = zeros + 0.0 - rhoel = zeros + gmn1 * p_jump - rhoer = zeros + gmn1 * 1.0 - - xinter = (shock_location + y_rel/np.sqrt(3.0) - + 2.0*shock_speed*t/np.sqrt(3.0)) + rhol = eos.gamma() * rho_jump + rhor = eos.gamma() + ul = up * np.cos(np.pi/6.0) + ur = 0.0 + vl = up * np.sin(np.pi/6.0) + vr = 0.0 + rhoel = gmn1 * p_jump + rhoer = gmn1 * 1.0 + + xinter = (self._shock_location + y_rel/np.sqrt(3.0) + + 2.0*self._shock_speed*t/np.sqrt(3.0)) sigma = 0.05 xtanh = 1.0/sigma*(x_rel-xinter) mass = rhol/2.0*(actx.np.tanh(-xtanh)+1.0)+rhor/2.0*(actx.np.tanh(xtanh)+1.0) @@ -374,6 +369,7 @@ def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): + rhoer/2.0*(actx.np.tanh(xtanh)+1.0)) u = ul/2.0*(actx.np.tanh(-xtanh)+1.0)+ur/2.0*(actx.np.tanh(xtanh)+1.0) v = vl/2.0*(actx.np.tanh(-xtanh)+1.0)+vr/2.0*(actx.np.tanh(xtanh)+1.0) + vel = make_obj_array([u, v]) mom = mass * vel energy = rhoe + .5*mass*np.dot(vel, vel) From fbe4de37231785c0583b079093aef024d6b66a7d Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Tue, 27 Apr 2021 12:37:12 -0500 Subject: [PATCH 064/385] Remove superfluous isinstance check Co-authored-by: Matt Smith --- mirgecom/artificial_viscosity.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 35db68006..f6374d4f0 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -110,15 +110,11 @@ def _facial_flux_q(discr, q_tpair): q_int = q_tpair.int actx = q_int[0].array_context - flux_dis = q_tpair.avg - if isinstance(flux_dis, np.ndarray): - flux_dis = flux_dis.reshape(-1, 1) - normal = thaw(actx, discr.normal(q_tpair.dd)) # This uses a central scalar flux along nhat: # flux = 1/2 * (Q- + Q+) * nhat - flux_out = flux_dis * normal + flux_out = np.outer(q_tpair.avg, normal) return discr.project(q_tpair.dd, "all_faces", flux_out) From 7dfa8143c5162dada86b5d819469fe53d42128ff Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Tue, 27 Apr 2021 10:39:44 -0700 Subject: [PATCH 065/385] Use added grudge functionality to get modal solution --- mirgecom/artificial_viscosity.py | 36 ++++---------------------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index f6374d4f0..a0e5daeb3 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -92,13 +92,12 @@ """ import numpy as np -import loopy as lp -from modepy import vandermonde from pytools.obj_array import obj_array_vectorize from meshmode.dof_array import thaw, DOFArray from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.eager import interior_trace_pair, cross_rank_trace_pairs from grudge.symbolic.primitives import TracePair +from grudge.dof_desc import DD_VOLUME_MODAL, DD_VOLUME from mirgecom.fluid import ( split_conserved, join_conserved_vectors @@ -230,22 +229,6 @@ def artificial_viscosity(discr, t, eos, boundaries, q, alpha, **kwargs): q=q, alpha=alpha, t=t) -def linear_operator_kernel(): - """Apply linear operator to all elements.""" - from meshmode.array_context import make_loopy_program - - knl = make_loopy_program( - """{[iel,idof,j]: - 0<=iel Date: Tue, 27 Apr 2021 10:52:00 -0700 Subject: [PATCH 066/385] Updated AV documentation to reflect new use of cv-specific fluid functions --- mirgecom/artificial_viscosity.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index a0e5daeb3..26b92cfcd 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -144,10 +144,9 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): Parameters ---------- q - State array which expects the quantity to be limited on to be listed - first in the array. For the Euler equations this could be the canonical - conserved variables (mass, energy, mometum) for the fluid along with a - vector of species masses for multi-component fluids. + State array of the canonical conserved variables (mass, energy, momentum) + for the fluid along with a vector of species masses for multi-component + fluids. boundaries Dictionary of boundary functions, one for each valid boundary tag From 88f35307ee612d81fac0d24237182a40ba39cd05 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Wed, 28 Apr 2021 10:53:59 -0500 Subject: [PATCH 067/385] remove commented out import Co-authored-by: Thomas H. Gibson --- mirgecom/boundary.py | 1 - 1 file changed, 1 deletion(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 65be0ed17..d44d8050e 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -35,7 +35,6 @@ import numpy as np from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa -# from mirgecom.eos import IdealSingleGas from grudge.symbolic.primitives import TracePair from mirgecom.fluid import ( split_conserved, From bde00282367898931ca1c926705440f72cb22e7d Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Wed, 28 Apr 2021 17:32:38 -0700 Subject: [PATCH 068/385] Fix sign error in doublemach case --- mirgecom/initializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 0caa5c6c5..6e27c6621 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -355,7 +355,7 @@ def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): rhor = eos.gamma() ul = up * np.cos(np.pi/6.0) ur = 0.0 - vl = up * np.sin(np.pi/6.0) + vl = - up * np.sin(np.pi/6.0) vr = 0.0 rhoel = gmn1 * p_jump rhoer = gmn1 * 1.0 From 13d26191e7fc618c59905bb5bf81f5eb307fd5f7 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 29 Apr 2021 14:22:15 -0500 Subject: [PATCH 069/385] Replace join_conserved_vectors with np.stack per @majosm suggestion. (#16) * Replace join_conserved_vectors with np.stack per @majosm suggestion. * Make it work for scalar Q * Clean up grad_q boundary routine after array shape corrections. * Update docs per @w-hagen review comments. * Nudge doc precision --- mirgecom/artificial_viscosity.py | 61 ++++++++++++++++++++------------ mirgecom/boundary.py | 26 ++++---------- test/test_av.py | 18 +++------- 3 files changed, 49 insertions(+), 56 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 26b92cfcd..3df91c417 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -98,42 +98,45 @@ from grudge.eager import interior_trace_pair, cross_rank_trace_pairs from grudge.symbolic.primitives import TracePair from grudge.dof_desc import DD_VOLUME_MODAL, DD_VOLUME -from mirgecom.fluid import ( - split_conserved, - join_conserved_vectors -) def _facial_flux_q(discr, q_tpair): """Compute facial flux for each scalar component of Q.""" - q_int = q_tpair.int - actx = q_int[0].array_context + flux_dis = q_tpair.avg + if isinstance(flux_dis, np.ndarray): + actx = flux_dis[0].array_context + flux_dis = flux_dis.reshape(-1, 1) + else: + actx = flux_dis.array_context normal = thaw(actx, discr.normal(q_tpair.dd)) # This uses a central scalar flux along nhat: # flux = 1/2 * (Q- + Q+) * nhat - flux_out = np.outer(q_tpair.avg, normal) + flux_out = flux_dis * normal return discr.project(q_tpair.dd, "all_faces", flux_out) def _facial_flux_r(discr, r_tpair): """Compute facial flux for vector component of grad(Q).""" - r_int = r_tpair.int - actx = r_int[0][0].array_context + flux_dis = r_tpair.avg + if isinstance(flux_dis[0], np.ndarray): + actx = flux_dis[0][0].array_context + else: + actx = flux_dis[0].array_context normal = thaw(actx, discr.normal(r_tpair.dd)) # This uses a central vector flux along nhat: # flux = 1/2 * (grad(Q)- + grad(Q)+) .dot. nhat - flux_out = r_tpair.avg @ normal + flux_out = flux_dis @ normal return discr.project(r_tpair.dd, "all_faces", flux_out) def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): - r"""Compute artificial viscosity for the Euler equations. + r"""Compute the artificial viscosity right-hand-side. Computes the the right-hand-side term for artificial viscosity. @@ -143,36 +146,48 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): Parameters ---------- - q - State array of the canonical conserved variables (mass, energy, momentum) + q: :class:`~meshmode.dof_array.DOFArray` or :class:`~numpy.ndarray` + + Single :class:`~meshmode.dof_array.DOFArray` for a scalar or an object array + (:class:`~numpy.ndarray`) for a vector of + :class:`~meshmode.dof_array.DOFArray` on which to operate. + + When used with fluid solvers, *q* is expected to be the fluid state array + of the canonical conserved variables (mass, energy, momentum) for the fluid along with a vector of species masses for multi-component fluids. - boundaries + boundaries: float + Dictionary of boundary functions, one for each valid boundary tag - t + t: float + Time - alpha + alpha: float + The maximum artificial viscosity coefficient to be applied - eos: mirgecom.eos.GasEOS + eos: :class:`~mirgecom.eos.GasEOS` + Only used as a pass through to the boundary conditions. Returns ------- numpy.ndarray + The artificial viscosity operator applied to *q*. """ - dim = discr.dim - cv = split_conserved(dim, q) - - # Get smoothness indicator based on fluid mass density - indicator = smoothness_indicator(discr, cv.mass, **kwargs) + # Get smoothness indicator based on first component + indicator_field = q[0] if isinstance(q, np.ndarray) else q + indicator = smoothness_indicator(discr, indicator_field, **kwargs) # R=Grad(Q) volume part - grad_q_vol = join_conserved_vectors(dim, obj_array_vectorize(discr.weak_grad, q)) + if isinstance(q, np.ndarray): + grad_q_vol = np.stack(obj_array_vectorize(discr.weak_grad, q), axis=0) + else: + grad_q_vol = discr.weak_grad(q) # R=Grad(Q) Q flux over interior faces q_flux_int = _facial_flux_q(discr, q_tpair=interior_trace_pair(discr, q)) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index d44d8050e..2d3cc292b 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -38,8 +38,7 @@ from grudge.symbolic.primitives import TracePair from mirgecom.fluid import ( split_conserved, - join_conserved, - join_conserved_vectors + join_conserved ) @@ -172,9 +171,9 @@ def exterior_grad_q(self, discr, grad_q, btag, **kwargs): """Get the exterior grad(Q) on the boundary.""" # Grab some boundary-relevant data num_equations, dim = grad_q.shape - num_species = num_equations - dim - 2 cv = split_conserved(dim, grad_q) actx = cv.mass[0].array_context + # Grab a unit normal to the boundary normal = thaw(actx, discr.normal(btag)) @@ -182,23 +181,12 @@ def exterior_grad_q(self, discr, grad_q, btag, **kwargs): int_soln = discr.project("vol", btag, grad_q) int_cv = split_conserved(dim, int_soln) - # create result array to fill - result = np.empty(shape=grad_q.shape, dtype=object) - - # flip signs on mass and energy - # to apply a neumann condition on q - result[0] = -int_cv.mass - result[1] = -int_cv.energy - # Subtract 2*wall-normal component of q # to enforce q=0 on the wall - # flip remaining components to set a neumann condition s_mom_normcomp = np.outer(normal, np.dot(int_cv.momentum, normal)) - s_mom_flux = 2*s_mom_normcomp - int_cv.momentum - for idim in range(dim): - result[2+idim] = s_mom_flux[idim] - - for ispec in range(num_species): - result[dim+2+ispec] = -int_cv.species_mass[ispec] + s_mom_flux = int_cv.momentum - 2*s_mom_normcomp - return join_conserved_vectors(dim, result) + # flip components to set a neumann condition + return join_conserved(dim, mass=-int_cv.mass, energy=-int_cv.energy, + momentum=-s_mom_flux, + species_mass=-int_cv.species_mass) diff --git a/test/test_av.py b/test/test_av.py index fa6a7e9e3..152d32a30 100644 --- a/test/test_av.py +++ b/test/test_av.py @@ -29,13 +29,9 @@ import numpy as np import pyopencl as cl import pytest -from pytools.obj_array import make_obj_array from meshmode.array_context import PyOpenCLArrayContext from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL -from mirgecom.fluid import ( - join_conserved, -) from mirgecom.artificial_viscosity import ( av_operator, smoothness_indicator @@ -183,28 +179,22 @@ def test_artificial_viscosity(ctx_factory, dim, order): boundaries = {BTAG_ALL: DummyBoundary()} # Uniform field return 0 rhs - mass = zeros + 1.0 - energy = zeros + 1.0 - momentum = make_obj_array([zeros + 0 for _ in range(dim)]) - q = join_conserved(dim, mass=mass, energy=energy, momentum=momentum) + q = zeros + 1.0 rhs = av_operator(discr, t=0, eos=None, boundaries=boundaries, q=q, alpha=1.0, s0=-np.inf) err = discr.norm(rhs, np.inf) assert err < tolerance # Linear field return 0 rhs - mass = nodes[0] - energy = 2.5 + zeros - q = join_conserved(dim, mass=mass, energy=energy, momentum=momentum) + q = nodes[0] rhs = av_operator(discr, t=0, eos=None, boundaries=boundaries, q=q, alpha=1.0, s0=-np.inf) err = discr.norm(rhs, np.inf) assert err < tolerance # Quadratic field return constant 2 - mass = np.dot(nodes, nodes) - q = join_conserved(dim, mass=mass, energy=energy, momentum=momentum) + q = np.dot(nodes, nodes) rhs = av_operator(discr, t=0, eos=None, boundaries=boundaries, q=q, alpha=1.0, s0=-np.inf) - err = discr.norm(2.*dim-rhs[0], np.inf) + err = discr.norm(2.*dim-rhs, np.inf) assert err < tolerance From b3556c9b210a17c867bdc1e720e93db26ddd2d31 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 29 Apr 2021 18:27:02 -0500 Subject: [PATCH 070/385] Rename vars more precisely in AV boundary routine. (#17) * Rename vars more precisely in AV boundary routine. * Correct mistaken variable identity. --- mirgecom/boundary.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 2d3cc292b..0bf4764f2 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -178,15 +178,15 @@ def exterior_grad_q(self, discr, grad_q, btag, **kwargs): normal = thaw(actx, discr.normal(btag)) # Get the interior soln - int_soln = discr.project("vol", btag, grad_q) - int_cv = split_conserved(dim, int_soln) + gradq_int = discr.project("vol", btag, grad_q) + gradq_comp = split_conserved(dim, gradq_int) # Subtract 2*wall-normal component of q # to enforce q=0 on the wall - s_mom_normcomp = np.outer(normal, np.dot(int_cv.momentum, normal)) - s_mom_flux = int_cv.momentum - 2*s_mom_normcomp + s_mom_normcomp = np.outer(normal, np.dot(gradq_comp.momentum, normal)) + s_mom_flux = gradq_comp.momentum - 2*s_mom_normcomp # flip components to set a neumann condition - return join_conserved(dim, mass=-int_cv.mass, energy=-int_cv.energy, + return join_conserved(dim, mass=-gradq_comp.mass, energy=-gradq_comp.energy, momentum=-s_mom_flux, - species_mass=-int_cv.species_mass) + species_mass=-gradq_comp.species_mass) From ef9c67e27592049e2ae1a134c9aeaa0eebdd85c3 Mon Sep 17 00:00:00 2001 From: "Thomas H. Gibson" Date: Thu, 29 Apr 2021 18:27:42 -0500 Subject: [PATCH 071/385] Make artificial viscosity documentation more general (#18) * Make artificial viscosity documentation more general * One line header for artificial viscosity module --- doc/operators/artificial_viscosity.rst | 4 ++++ doc/operators/gas-dynamics.rst | 1 - doc/operators/operators.rst | 1 + mirgecom/artificial_viscosity.py | 27 ++++++++++++++------------ 4 files changed, 20 insertions(+), 13 deletions(-) create mode 100644 doc/operators/artificial_viscosity.rst diff --git a/doc/operators/artificial_viscosity.rst b/doc/operators/artificial_viscosity.rst new file mode 100644 index 000000000..9b583612a --- /dev/null +++ b/doc/operators/artificial_viscosity.rst @@ -0,0 +1,4 @@ +Artificial Viscosity +==================== + +.. automodule:: mirgecom.artificial_viscosity diff --git a/doc/operators/gas-dynamics.rst b/doc/operators/gas-dynamics.rst index 48aa91507..d1949554a 100644 --- a/doc/operators/gas-dynamics.rst +++ b/doc/operators/gas-dynamics.rst @@ -8,4 +8,3 @@ Gas Dynamics .. automodule:: mirgecom.flux .. automodule:: mirgecom.boundary .. automodule:: mirgecom.euler -.. automodule:: mirgecom.artificial_viscosity diff --git a/doc/operators/operators.rst b/doc/operators/operators.rst index c0b40c7fe..0907c286d 100644 --- a/doc/operators/operators.rst +++ b/doc/operators/operators.rst @@ -6,4 +6,5 @@ Operators wave-eq diffusion + artificial_viscosity gas-dynamics diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 3df91c417..5a134c01a 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -1,27 +1,30 @@ -r""":mod:`mirgecom.artificial_viscosity` Artificial viscocity for Euler. +r""":mod:`mirgecom.artificial_viscosity` Artificial viscosity for hyperbolic systems. -Euler Equations with artificial viscosity term: +Consider the following system of conservation laws of the form: .. math:: - \partial_t \mathbf{Q} = -\nabla\cdot\mathbf{F}^I + - \nabla\cdot{\varepsilon\nabla\mathbf{Q}} + \partial_t \mathbf{Q} + \nabla\cdot\mathbf{F} = 0, -where: +where $\mathbf{Q}$ is the state vector and $\mathbf{F}$ is the vector of +fluxes. This module applies an artificial viscosity term by augmenting +the governing equations in the following way: + +.. math:: -- fluid state: $\mathbf{Q} = [\rho, \rho{E}, \rho\mathbf{V}, \rho\mathbf{Y}]$ -- inviscid fluxes: $\mathbf{F}^I$ -- artifical viscosity coefficient: $\varepsilon$ + \partial_t \mathbf{Q} + \nabla\cdot\mathbf{F} = + \nabla\cdot{\varepsilon\nabla\mathbf{Q}}, -To evalutate the second order derivative the problem is recast as a set of first - order problems: +where $\varepsilon$ is the artificial viscosity coefficient. +To evalutate the second order derivative numerically, the problem +is recast as a set of first order problems: .. math:: - \partial_t{\mathbf{Q}} &= \nabla\cdot\mathbf{R} -\nabla\cdot\mathbf{F}^I \\ + \partial_t{\mathbf{Q}} + \nabla\cdot\mathbf{F} &= \nabla\cdot\mathbf{R} \\ \mathbf{R} &= \varepsilon\nabla\mathbf{Q} -where $\mathbf{R}$ is an intermediate variable, and the artitifial viscosity +where $\mathbf{R}$ is an auxiliary variable, and the artitifial viscosity coefficient, $\varepsilon$, is spatially dependent and calculated using the smoothness indicator of [Persson_2012]_: From 961a5bb42156d36887f5cd671512a165c607fd3e Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Thu, 29 Apr 2021 18:31:23 -0500 Subject: [PATCH 072/385] Fixed spelling Co-authored-by: Matt Smith --- mirgecom/artificial_viscosity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 5a134c01a..4001d4e34 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -24,7 +24,7 @@ \partial_t{\mathbf{Q}} + \nabla\cdot\mathbf{F} &= \nabla\cdot\mathbf{R} \\ \mathbf{R} &= \varepsilon\nabla\mathbf{Q} -where $\mathbf{R}$ is an auxiliary variable, and the artitifial viscosity +where $\mathbf{R}$ is an auxiliary variable, and the artificial viscosity coefficient, $\varepsilon$, is spatially dependent and calculated using the smoothness indicator of [Persson_2012]_: @@ -40,7 +40,7 @@ - $u_{N_{p-1}}$ is the truncated modal represention to the polynomial order $p-1$ - The $L_2$ inner product on an element is denoted $\langle \cdot,\cdot \rangle_e$ -The elementwise viscoisty is then calculated: +The elementwise viscosity is then calculated: .. math:: From ea3dc359abbd2e8f852b4f60ff4a77dbec3e0ac2 Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Thu, 29 Apr 2021 22:32:11 -0500 Subject: [PATCH 073/385] Remove join_conserved_vectors Co-authored-by: Mike Campbell --- mirgecom/fluid.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/mirgecom/fluid.py b/mirgecom/fluid.py index ef2e64909..655e2abe8 100644 --- a/mirgecom/fluid.py +++ b/mirgecom/fluid.py @@ -6,7 +6,6 @@ .. autoclass:: ConservedVars .. autofunction:: split_conserved .. autofunction:: join_conserved -.. autofunction:: join_conserved_vectors Helper Functions ^^^^^^^^^^^^^^^^ @@ -300,21 +299,6 @@ def join_conserved(dim, mass, energy, momentum, return result -def join_conserved_vectors(dim, ary): - r"""Create a 2D array of shape(len(ary), dim).""" - neq = len(ary) - nspecies = neq - (dim+2) - retval = np.empty(shape=(neq, dim), dtype=object) - cv = split_conserved(dim, ary) - retval[0] = cv.mass - retval[1] = cv.energy - for i in range(dim): - retval[2+i] = ary[2+i] - for i in range(nspecies): - retval[dim+2+i] = ary[dim+2+i] - return retval - - def velocity_gradient(discr, cv, grad_cv): r""" Compute the gradient of fluid velocity. From 70a469664a417f211dd1dd4126ecaf6bb6644c11 Mon Sep 17 00:00:00 2001 From: "Thomas H. Gibson" Date: Thu, 6 May 2021 14:13:04 -0500 Subject: [PATCH 074/385] Rewrite shock indicator, cache program and mode computations (#20) --- mirgecom/artificial_viscosity.py | 80 +++++++++++++++++--------------- 1 file changed, 43 insertions(+), 37 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 4001d4e34..cf1cd8ae3 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -95,6 +95,8 @@ """ import numpy as np + +from pytools import memoize_in, keyed_memoize_in from pytools.obj_array import obj_array_vectorize from meshmode.dof_array import thaw, DOFArray from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa @@ -246,24 +248,6 @@ def artificial_viscosity(discr, t, eos, boundaries, q, alpha, **kwargs): q=q, alpha=alpha, t=t) -def compute_smoothness_indicator(): - """Compute the smoothness indicator for all elements.""" - from meshmode.array_context import make_loopy_program - - knl = make_loopy_program( - """{[iel,idof,j,k]: - 0<=iel Date: Thu, 6 May 2021 14:20:31 -0500 Subject: [PATCH 075/385] Update double mach example (#22) --- examples/doublemach-mpi.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 17d538466..e58ea34c6 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -1,4 +1,4 @@ -"""Demonstrate doublemach reflection.""" +"""Demonstrate double mach reflection.""" __copyright__ = """ Copyright (C) 2020 University of Illinois Board of Trustees @@ -31,6 +31,7 @@ from meshmode.array_context import PyOpenCLArrayContext from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa +from grudge.dof_desc import DTAG_BOUNDARY from grudge.eager import EagerDGDiscretization from grudge.shortcuts import make_visualizer @@ -124,14 +125,13 @@ def main(ctx_factory=cl.create_some_context): eos = IdealSingleGas() initializer = DoubleMachReflection() casename = "doubleMach" - from grudge import sym boundaries = { - sym.DTAG_BOUNDARY("ic1"): PrescribedBoundary(initializer), - sym.DTAG_BOUNDARY("ic2"): PrescribedBoundary(initializer), - sym.DTAG_BOUNDARY("ic3"): PrescribedBoundary(initializer), - sym.DTAG_BOUNDARY("wall"): AdiabaticSlipBoundary(), - sym.DTAG_BOUNDARY("out"): AdiabaticSlipBoundary(), + DTAG_BOUNDARY("ic1"): PrescribedBoundary(initializer), + DTAG_BOUNDARY("ic2"): PrescribedBoundary(initializer), + DTAG_BOUNDARY("ic3"): PrescribedBoundary(initializer), + DTAG_BOUNDARY("wall"): AdiabaticSlipBoundary(), + DTAG_BOUNDARY("out"): AdiabaticSlipBoundary(), } constant_cfl = False nstatus = 10 From 445c5c81dbf1f2e9777ec1593f1fe60713bff3ee Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Thu, 6 May 2021 14:25:18 -0500 Subject: [PATCH 076/385] Clean up object array handling in flux routines (#19) * clean up object array handling in flux routines * add note about get_array_container_context --- mirgecom/artificial_viscosity.py | 38 +++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index cf1cd8ae3..273d7fb74 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -105,37 +105,49 @@ from grudge.dof_desc import DD_VOLUME_MODAL, DD_VOLUME +# FIXME: Remove when get_array_container_context is added to meshmode +def _get_actx(obj): + if isinstance(obj, TracePair): + return _get_actx(obj.int) + if isinstance(obj, np.ndarray): + return _get_actx(obj[0]) + elif isinstance(obj, DOFArray): + return obj.array_context + else: + raise ValueError("Unknown type; can't retrieve array context.") + + +# Tweak the behavior of np.outer to return a lower-dimensional object if either/both +# of the arguments are scalars (np.outer always returns a matrix) +def _outer(a, b): + if isinstance(a, np.ndarray) and isinstance(b, np.ndarray): + return np.outer(a, b) + else: + return a*b + + def _facial_flux_q(discr, q_tpair): """Compute facial flux for each scalar component of Q.""" - flux_dis = q_tpair.avg - if isinstance(flux_dis, np.ndarray): - actx = flux_dis[0].array_context - flux_dis = flux_dis.reshape(-1, 1) - else: - actx = flux_dis.array_context + actx = _get_actx(q_tpair) normal = thaw(actx, discr.normal(q_tpair.dd)) # This uses a central scalar flux along nhat: # flux = 1/2 * (Q- + Q+) * nhat - flux_out = flux_dis * normal + flux_out = _outer(q_tpair.avg, normal) return discr.project(q_tpair.dd, "all_faces", flux_out) def _facial_flux_r(discr, r_tpair): """Compute facial flux for vector component of grad(Q).""" - flux_dis = r_tpair.avg - if isinstance(flux_dis[0], np.ndarray): - actx = flux_dis[0][0].array_context - else: - actx = flux_dis[0].array_context + actx = _get_actx(r_tpair) normal = thaw(actx, discr.normal(r_tpair.dd)) # This uses a central vector flux along nhat: # flux = 1/2 * (grad(Q)- + grad(Q)+) .dot. nhat - flux_out = flux_dis @ normal + flux_out = r_tpair.avg @ normal return discr.project(r_tpair.dd, "all_faces", flux_out) From e2d7dddfc99ee5b6887408ebd44d2904c6bf9503 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 7 May 2021 15:03:35 -0500 Subject: [PATCH 077/385] Sooth the bugbear. --- mirgecom/initializers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 2a82c1d86..1152f018c 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -319,7 +319,7 @@ def __init__( self._shock_location = shock_location self._shock_speed = shock_speed - def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): + def __call__(self, x_vec, *, t=0, eos=None, **kwargs): r""" Create double mach reflection solution at locations *x_vec*. @@ -341,6 +341,8 @@ def __call__(self, x_vec, *, t=0, eos=IdealSingleGas()): # Fail if numdim is other than 2 if(len(x_vec)) != 2: raise ValueError("Case only defined for 2 dimensions") + if eos is None: + eos=IdealSingleGas() gm1 = eos.gamma() - 1.0 gp1 = eos.gamma() + 1.0 From 3a61993e026a7f45fc55987e64077eaf5c8b0324 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 7 May 2021 15:09:01 -0500 Subject: [PATCH 078/385] Unpoke the bugbear. --- mirgecom/initializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 1152f018c..34512a2bb 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -342,7 +342,7 @@ def __call__(self, x_vec, *, t=0, eos=None, **kwargs): if(len(x_vec)) != 2: raise ValueError("Case only defined for 2 dimensions") if eos is None: - eos=IdealSingleGas() + eos = IdealSingleGas() gm1 = eos.gamma() - 1.0 gp1 = eos.gamma() + 1.0 From 3b88176c5f1c2f9edc1dfcf8730273bdff5db18b Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 10 May 2021 14:07:21 -0500 Subject: [PATCH 079/385] Pull AV boundaries into NS. --- mirgecom/artificial_viscosity.py | 48 +++++--------- mirgecom/boundary.py | 106 +++++++++++++++++++++++++++---- mirgecom/euler.py | 2 +- mirgecom/flux.py | 31 +++++++++ 4 files changed, 141 insertions(+), 46 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 273d7fb74..d0977e837 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -206,22 +206,15 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): else: grad_q_vol = discr.weak_grad(q) - # R=Grad(Q) Q flux over interior faces - q_flux_int = _facial_flux_q(discr, q_tpair=interior_trace_pair(discr, q)) - # R=Grad(Q) Q flux interior faces on partition boundaries - q_flux_pb = sum(_facial_flux_q(discr, q_tpair=pb_tpair) - for pb_tpair in cross_rank_trace_pairs(discr, q)) - # R=Grad(Q) Q flux domain boundary part (i.e. BCs) - q_flux_db = 0 - for btag in boundaries: - q_tpair = TracePair( - btag, - interior=discr.project("vol", btag, q), - exterior=boundaries[btag].exterior_q(discr, btag=btag, t=t, - q=q, eos=eos)) - q_flux_db = q_flux_db + _facial_flux_q(discr, q_tpair=q_tpair) - # Total Q flux across element boundaries - q_bnd_flux = q_flux_int + q_flux_pb + q_flux_db + # Total flux of fluid soln Q across element boundaries + q_bnd_flux = (_facial_flux_q(discr, q_tpair=interior_trace_pair(discr, q)) + + sum(_facial_flux_q(discr, q_tpair=pb_tpair) + for pb_tpair in cross_rank_trace_pairs(discr, q))) + q_bnd_flux2 = sum(bnd.q_boundary_flux(discr, btag, q=q, eos=eos, **kwargs) + for btag, bnd in boundaries.items()) + if isinstance(q, np.ndarray): + q_bnd_flux2 = np.stack(q_bnd_flux2) + q_bnd_flux = q_bnd_flux + q_bnd_flux2 # Compute R r = discr.inverse_mass( @@ -230,25 +223,16 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): # RHS_av = div(R) volume part div_r_vol = discr.weak_div(r) - # RHS_av = div(R): grad(Q) flux interior faces part - r_flux_int = _facial_flux_r(discr, r_tpair=interior_trace_pair(discr, r)) - # RHS_av = div(R): grad(Q) flux interior faces on the partition boundaries - r_flux_pb = sum(_facial_flux_r(discr, r_tpair=pb_tpair) + # Total flux of grad(Q) across element boundaries + r_bnd_flux = (_facial_flux_r(discr, r_tpair=interior_trace_pair(discr, r)) + + sum(_facial_flux_r(discr, r_tpair=pb_tpair) for pb_tpair in cross_rank_trace_pairs(discr, r)) - # RHS_av = div(R): grad(Q) flux domain boundary part (BCs) - r_flux_db = 0 - for btag in boundaries: - r_tpair = TracePair( - btag, - interior=discr.project("vol", btag, r), - exterior=boundaries[btag].exterior_grad_q(discr, btag=btag, t=t, - grad_q=r, eos=eos)) - r_flux_db = r_flux_db + _facial_flux_r(discr, r_tpair=r_tpair) - # Total grad(Q) flux element boundaries - r_flux_bnd = r_flux_int + r_flux_pb + r_flux_db + + sum(bnd.s_boundary_flux(discr, btag, grad_q=r, eos=eos, + **kwargs) + for btag, bnd in boundaries.items())) # Return the AV RHS term - return discr.inverse_mass(-div_r_vol + discr.face_mass(r_flux_bnd)) + return discr.inverse_mass(-div_r_vol + discr.face_mass(r_bnd_flux)) def artificial_viscosity(discr, t, eos, boundaries, q, alpha, **kwargs): diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 61de39d31..bd681a3a9 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -98,7 +98,7 @@ class FluidBC(FluidBoundary): .. automethod:: boundary_pair """ - def q_boundary_flux(self, discr, btag, eos, q, **kwargs): + def q_boundary_flux(self, discr, btag, q, eos, **kwargs): """Get the flux through boundary *btag* for each scalar in *q*.""" raise NotImplementedError() @@ -106,19 +106,19 @@ def s_boundary_flux(self, discr, btag, q, eos, **kwargs): r"""Get $\nabla\mathbf{Q}$ flux across the boundary faces.""" raise NotImplementedError() - def t_boundary_flux(self, discr, btag, eos, q, **kwargs): + def t_boundary_flux(self, discr, btag, q, eos, **kwargs): """Get the "temperature flux" through boundary *btag*.""" raise NotImplementedError() - def inviscid_boundary_flux(self, discr, btag, eos, q, **kwargs): + def inviscid_boundary_flux(self, discr, btag, q, eos, **kwargs): """Get the inviscid part of the physical flux across the boundary *btag*.""" raise NotImplementedError() - def viscous_boundary_flux(self, discr, btag, eos, q, grad_q, grad_t, **kwargs): + def viscous_boundary_flux(self, discr, btag, q, grad_q, grad_t, eos, **kwargs): """Get the viscous part of the physical flux across the boundary *btag*.""" raise NotImplementedError() - def boundary_pair(self, discr, btag, eos, u, **kwargs): + def boundary_pair(self, discr, btag, u, eos, **kwargs): """Get the interior and exterior solution (*u*) on the boundary.""" raise NotImplementedError() @@ -132,7 +132,10 @@ class PrescribedInviscidBoundary(FluidBC): """ def __init__(self, inviscid_boundary_flux_func=None, boundary_pair_func=None, - inviscid_facial_flux_func=None, fluid_solution_func=None): + inviscid_facial_flux_func=None, fluid_solution_func=None, + fluid_solution_flux_func=None, scalar_numerical_flux_func=None, + fluid_solution_gradient_func=None, + fluid_solution_gradient_flux_func=None): """Initialize the PrescribedInviscidBoundary and methods.""" self._bnd_pair_func = boundary_pair_func self._inviscid_bnd_flux_func = inviscid_boundary_flux_func @@ -140,8 +143,25 @@ def __init__(self, inviscid_boundary_flux_func=None, boundary_pair_func=None, if not self._inviscid_facial_flux_func: self._inviscid_facial_flux_func = inviscid_facial_flux self._fluid_soln_func = fluid_solution_func + self._fluid_soln_flux_func = fluid_solution_flux_func + self._scalar_num_flux_func = scalar_numerical_flux_func + from mirgecom.flux import central_scalar_flux + if not self._scalar_num_flux_func: + self._scalar_num_flux_func = central_scalar_flux + self._fluid_soln_grad_func = fluid_solution_gradient_func + self._fluid_soln_grad_flux_func = fluid_solution_gradient_flux_func + from mirgecom.flux import central_vector_flux + if not self._fluid_soln_grad_flux_func: + self._fluid_soln_grad_flux_func = central_vector_flux + + def _boundary_quantity(self, discr, btag, quantity, **kwargs): + """Get a boundary quantity on local boundary, or projected to "all_faces".""" + if "local" in kwargs: + if kwargs["local"]: + return quantity + return discr.project(btag, "all_faces", quantity) - def boundary_pair(self, discr, q, btag, **kwargs): + def boundary_pair(self, discr, btag, q, **kwargs): """Get the interior and exterior solution on the boundary.""" if self._bnd_pair_func: return self._bnd_pair_func(discr, q=q, btag=btag, **kwargs) @@ -165,9 +185,53 @@ def inviscid_boundary_flux(self, discr, btag, q, eos, **kwargs): int_soln = discr.project("vol", btag, q) return self._inviscid_bnd_flux_func(nodes, normal=nhat, q=int_soln, eos=eos, **kwargs) - bnd_tpair = self.boundary_pair(discr, q, btag, eos=eos, **kwargs) + bnd_tpair = self.boundary_pair(discr, btag=btag, q=q, eos=eos, **kwargs) return self._inviscid_facial_flux_func(discr, eos=eos, q_tpair=bnd_tpair) + def q_boundary_flux(self, discr, btag, q, **kwargs): + """Get the flux through boundary *btag* for each scalar in *q*.""" + if isinstance(q, np.ndarray): + actx = q[0].array_context + else: + actx = q.array_context + boundary_discr = discr.discr_from_dd(btag) + nodes = thaw(actx, boundary_discr.nodes()) + nhat = thaw(actx, discr.normal(btag)) + if self._fluid_soln_flux_func: + q_minus = discr.project("vol", btag, q) + flux_weak = self._fluid_soln_flux_func(nodes, q=q_minus, nhat=nhat, + **kwargs) + else: + bnd_pair = self.boundary_pair(discr, btag=btag, q=q, **kwargs) + flux_weak = self._scalar_num_flux_func(bnd_pair, normal=nhat) + + return self._boundary_quantity(discr, btag=btag, quantity=flux_weak, + **kwargs) + + def s_boundary_flux(self, discr, btag, grad_q, **kwargs): + r"""Get $\nabla\mathbf{Q}$ flux across the boundary faces.""" + grad_shape = grad_q.shape + if len(grad_shape) > 1: + actx = grad_q[0][0].array_context + else: + actx = grad_q[0].array_context + + boundary_discr = discr.discr_from_dd(btag) + nodes = thaw(actx, boundary_discr.nodes()) + nhat = thaw(actx, discr.normal(btag)) + grad_q_minus = discr.project("vol", btag, grad_q) + if self._fluid_soln_grad_func: + grad_q_plus = self._fluid_soln_grad_func(nodes, nhat=nhat, + grad_q=grad_q_minus, **kwargs) + else: + grad_q_plus = grad_q_minus + bnd_grad_pair = TracePair(btag, interior=grad_q_minus, exterior=grad_q_plus) + + return self._boundary_quantity( + discr, btag, self._fluid_soln_grad_flux_func(bnd_grad_pair, nhat), + **kwargs + ) + class PrescribedBoundary(PrescribedInviscidBoundary): """Boundary condition prescribes boundary soln with user-specified function. @@ -212,10 +276,6 @@ def exterior_q(self, discr, q, btag, **kwargs): dir_soln = discr.project("vol", btag, q) return dir_soln - def exterior_grad_q(self, discr, grad_q, btag, **kwargs): - """Get the grad_q on the exterior of the boundary.""" - return discr.project("vol", btag, grad_q) - class AdiabaticSlipBoundary(PrescribedInviscidBoundary): r"""Boundary condition implementing inviscid slip boundary. @@ -238,7 +298,9 @@ class AdiabaticSlipBoundary(PrescribedInviscidBoundary): def __init__(self): """Initialize AdiabaticSlipBoundary.""" PrescribedInviscidBoundary.__init__( - self, boundary_pair_func=self.adiabatic_slip_pair) + self, boundary_pair_func=self.adiabatic_slip_pair, + fluid_solution_gradient_func=self.exterior_grad_q + ) def adiabatic_slip_pair(self, discr, q, btag, **kwargs): """Get the interior and exterior solution on the boundary. @@ -279,6 +341,24 @@ def adiabatic_slip_pair(self, discr, q, btag, **kwargs): return TracePair(btag, interior=int_soln, exterior=bndry_soln) + def exterior_grad_q(self, nodes, nhat, grad_q, **kwargs): + """Get the exterior grad(Q) on the boundary.""" + # Grab some boundary-relevant data + num_equations, dim = grad_q.shape + + # Get the interior soln + gradq_comp = split_conserved(dim, grad_q) + + # Subtract 2*wall-normal component of q + # to enforce q=0 on the wall + s_mom_normcomp = np.outer(nhat, np.dot(gradq_comp.momentum, nhat)) + s_mom_flux = gradq_comp.momentum - 2*s_mom_normcomp + + # flip components to set a neumann condition + return join_conserved(dim, mass=-gradq_comp.mass, energy=-gradq_comp.energy, + momentum=-s_mom_flux, + species_mass=-gradq_comp.species_mass) + class IsothermalNoSlipBoundary(FluidBC): r"""Isothermal no-slip viscous wall boundary. diff --git a/mirgecom/euler.py b/mirgecom/euler.py index 2123843bc..0bef5679f 100644 --- a/mirgecom/euler.py +++ b/mirgecom/euler.py @@ -108,7 +108,7 @@ def euler_operator(discr, eos, boundaries, q, t=0.0): inviscid_facial_flux(discr, eos=eos, q_tpair=interior_trace_pair(discr, q)) + sum(inviscid_facial_flux(discr, eos=eos, q_tpair=part_tpair) for part_tpair in cross_rank_trace_pairs(discr, q)) - + sum(boundaries[btag].inviscid_boundary_flux(discr, btag, eos=eos, q=q) + + sum(boundaries[btag].inviscid_boundary_flux(discr, btag=btag, q=q, eos=eos) for btag in boundaries) ) diff --git a/mirgecom/flux.py b/mirgecom/flux.py index cbe911cf6..99695e59d 100644 --- a/mirgecom/flux.py +++ b/mirgecom/flux.py @@ -72,6 +72,37 @@ def central_scalar_flux(trace_pair, normal): return trace_pair.avg*normal +def central_vector_flux(trace_pair, normal): + r"""Compute a central vector flux. + + The central vector flux, $h$, is calculated as: + + .. math:: + + h(\mathbf{v}^-, \mathbf{v}^+; \mathbf{n}) = \frac{1}{2} + \left(\mathbf{v}^{+}+\mathbf{v}^{-}\right) \cdot \hat{n} + + where $\mathbf{v}^-, \matbhf{v}^+$, are the vectors on the interior and exterior + of the face across which the central flux is to be calculated, and $\hat{n}$ is + the unit normal to the face. + + Parameters + ---------- + trace_pair: `grudge.sym.TracePair` + Trace pair for the face upon which flux calculation is to be performed + normal: numpy.ndarray + object array of :class:`meshmode.dof_array.DOFArray` with outward-pointing + normals + + Returns + ------- + numpy.ndarray + object array of `meshmode.dof_array.DOFArray` with the central scalar flux + for each scalar component. + """ + return trace_pair.avg@normal + + def lfr_flux(q_tpair, f_tpair, normal, lam): r"""Compute Lax-Friedrichs/Rusanov flux after [Hesthaven_2008]_, Section 6.6. From 59df6df08c6d782443f685a1761ee933b4199e77 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Mon, 10 May 2021 19:33:13 -0500 Subject: [PATCH 080/385] AV misc fixes (#23) * remove call to deprecated create_parallel_grid * remove use of deprecated n parameter * add boundary_kwargs to av_operator * raise exception instead of asserting --- examples/doublemach-mpi.py | 7 ++++--- mirgecom/artificial_viscosity.py | 30 +++++++++++++++--------------- test/test_av.py | 11 ++++------- 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index e58ea34c6..18e6cee49 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -44,7 +44,7 @@ from mirgecom.simutil import ( inviscid_sim_timestep, sim_checkpoint, - create_parallel_grid, + generate_and_distribute_mesh, ExactSolutionMismatch, ) from mirgecom.io import make_init_message @@ -151,7 +151,7 @@ def main(ctx_factory=cl.create_some_context): gen_grid = partial(get_doublemach_mesh) - local_mesh, global_nelements = create_parallel_grid(comm, gen_grid) + local_mesh, global_nelements = generate_and_distribute_mesh(comm, gen_grid) local_nelements = local_mesh.nelements @@ -198,7 +198,8 @@ def my_rhs(t, state): return inviscid_operator( discr, q=state, t=t, boundaries=boundaries, eos=eos ) + av_operator( - discr, t=t, q=state, boundaries=boundaries, alpha=alpha, eos=eos, + discr, q=state, boundaries=boundaries, + boundary_kwargs={"t": t, "eos": eos}, alpha=alpha, s0=s0, kappa=kappa ) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 273d7fb74..cbc5f0bce 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -152,7 +152,7 @@ def _facial_flux_r(discr, r_tpair): return discr.project(r_tpair.dd, "all_faces", flux_out) -def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): +def av_operator(discr, boundaries, q, alpha, boundary_kwargs=None, **kwargs): r"""Compute the artificial viscosity right-hand-side. Computes the the right-hand-side term for artificial viscosity. @@ -178,17 +178,13 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): Dictionary of boundary functions, one for each valid boundary tag - t: float - - Time - alpha: float - The maximum artificial viscosity coefficient to be applied + The maximum artificial viscosity coefficient to be applied - eos: :class:`~mirgecom.eos.GasEOS` + boundary_kwargs: :class:`dict` - Only used as a pass through to the boundary conditions. + dictionary of extra arguments to pass through to the boundary conditions Returns ------- @@ -196,6 +192,9 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): The artificial viscosity operator applied to *q*. """ + if boundary_kwargs is None: + boundary_kwargs = dict() + # Get smoothness indicator based on first component indicator_field = q[0] if isinstance(q, np.ndarray) else q indicator = smoothness_indicator(discr, indicator_field, **kwargs) @@ -217,8 +216,8 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): q_tpair = TracePair( btag, interior=discr.project("vol", btag, q), - exterior=boundaries[btag].exterior_q(discr, btag=btag, t=t, - q=q, eos=eos)) + exterior=boundaries[btag].exterior_q(discr, btag=btag, q=q, + **boundary_kwargs)) q_flux_db = q_flux_db + _facial_flux_q(discr, q_tpair=q_tpair) # Total Q flux across element boundaries q_bnd_flux = q_flux_int + q_flux_pb + q_flux_db @@ -241,8 +240,8 @@ def av_operator(discr, t, eos, boundaries, q, alpha, **kwargs): r_tpair = TracePair( btag, interior=discr.project("vol", btag, r), - exterior=boundaries[btag].exterior_grad_q(discr, btag=btag, t=t, - grad_q=r, eos=eos)) + exterior=boundaries[btag].exterior_grad_q(discr, btag=btag, grad_q=r, + **boundary_kwargs)) r_flux_db = r_flux_db + _facial_flux_r(discr, r_tpair=r_tpair) # Total grad(Q) flux element boundaries r_flux_bnd = r_flux_int + r_flux_pb + r_flux_db @@ -256,8 +255,8 @@ def artificial_viscosity(discr, t, eos, boundaries, q, alpha, **kwargs): from warnings import warn warn("Do not call artificial_viscosity; it is now called av_operator. This" "function will disappear in 2021", DeprecationWarning, stacklevel=2) - return av_operator(discr=discr, eos=eos, boundaries=boundaries, - q=q, alpha=alpha, t=t) + return av_operator(discr=discr, boundaries=boundaries, + boundary_kwargs={"t": t, "eos": eos}, q=q, alpha=alpha, **kwargs) def smoothness_indicator(discr, u, kappa=1.0, s0=-6.0): @@ -281,7 +280,8 @@ def smoothness_indicator(discr, u, kappa=1.0, s0=-6.0): The elementwise constant values between 0 and 1 which indicate the smoothness of a given element. """ - assert isinstance(u, DOFArray) + if not isinstance(u, DOFArray): + raise ValueError("u argument must be a DOFArray.") actx = u.array_context diff --git a/test/test_av.py b/test/test_av.py index 152d32a30..0390bc960 100644 --- a/test/test_av.py +++ b/test/test_av.py @@ -169,7 +169,7 @@ def test_artificial_viscosity(ctx_factory, dim, order): from meshmode.mesh.generation import generate_regular_rect_mesh mesh = generate_regular_rect_mesh( - a=(-1.0, )*dim, b=(1.0, )*dim, n=(nel_1d, ) * dim + a=(-1.0, )*dim, b=(1.0, )*dim, nelements_per_axis=(nel_1d, )*dim ) discr = EagerDGDiscretization(actx, mesh, order=order) @@ -180,21 +180,18 @@ def test_artificial_viscosity(ctx_factory, dim, order): # Uniform field return 0 rhs q = zeros + 1.0 - rhs = av_operator(discr, t=0, eos=None, boundaries=boundaries, - q=q, alpha=1.0, s0=-np.inf) + rhs = av_operator(discr, boundaries=boundaries, q=q, alpha=1.0, s0=-np.inf) err = discr.norm(rhs, np.inf) assert err < tolerance # Linear field return 0 rhs q = nodes[0] - rhs = av_operator(discr, t=0, eos=None, boundaries=boundaries, - q=q, alpha=1.0, s0=-np.inf) + rhs = av_operator(discr, boundaries=boundaries, q=q, alpha=1.0, s0=-np.inf) err = discr.norm(rhs, np.inf) assert err < tolerance # Quadratic field return constant 2 q = np.dot(nodes, nodes) - rhs = av_operator(discr, t=0, eos=None, boundaries=boundaries, - q=q, alpha=1.0, s0=-np.inf) + rhs = av_operator(discr, boundaries=boundaries, q=q, alpha=1.0, s0=-np.inf) err = discr.norm(2.*dim-rhs, np.inf) assert err < tolerance From c0578f26c0a956da648bc3c298b563b511dfef2a Mon Sep 17 00:00:00 2001 From: Wyatt Hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 10 May 2021 17:49:55 -0700 Subject: [PATCH 081/385] Rename modes_active_flag --- mirgecom/artificial_viscosity.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index cbc5f0bce..cb677d1f2 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -296,11 +296,11 @@ def indicator_prg(): "{[kdof]: 0 <= kdof < ndiscr_nodes_in}" ], """ - result[iel,idof] = sum(kdof, vec[iel, kdof] \ - * vec[iel, kdof] \ - * modes[kdof]) / \ - sum(jdof, vec[iel, jdof] \ - * vec[iel, jdof] \ + result[iel,idof] = sum(kdof, vec[iel, kdof] \ + * vec[iel, kdof] \ + * modes_active_flag[kdof]) / \ + sum(jdof, vec[iel, jdof] \ + * vec[iel, jdof] \ + 1.0e-12 / ndiscr_nodes_in) """, name="smooth_comp", @@ -327,7 +327,7 @@ def highest_mode(grp): actx.call_loopy( indicator_prg(), vec=uhat[grp.index], - modes=highest_mode(grp))["result"] + modes_active_flag=highest_mode(grp))["result"] for grp in discr.discr_from_dd("vol").groups ) ) From 893fd224a5c8313ec8e1f4434bee2261e5c8090d Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 11 May 2021 08:55:09 -0500 Subject: [PATCH 082/385] Account for slight interface change in tests. --- test/test_bc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_bc.py b/test/test_bc.py index 13321cf7b..fa761fc7a 100644 --- a/test/test_bc.py +++ b/test/test_bc.py @@ -91,8 +91,8 @@ def test_slipwall_identity(actx_factory, dim): from functools import partial bnd_norm = partial(discr.norm, p=np.inf, dd=BTAG_ALL) - bnd_pair = wall.boundary_pair(discr, uniform_state, t=0.0, - btag=BTAG_ALL, eos=eos) + bnd_pair = wall.boundary_pair(discr, btag=BTAG_ALL, q=uniform_state, + eos=eos, t=0.0) bnd_cv_int = split_conserved(dim, bnd_pair.int) bnd_cv_ext = split_conserved(dim, bnd_pair.ext) @@ -156,8 +156,8 @@ def test_slipwall_flux(actx_factory, dim, order): from mirgecom.initializers import Uniform initializer = Uniform(dim=dim, velocity=vel) uniform_state = initializer(nodes) - bnd_pair = wall.boundary_pair(discr, uniform_state, t=0.0, - btag=BTAG_ALL, eos=eos) + bnd_pair = wall.boundary_pair(discr, btag=BTAG_ALL, q=uniform_state, + eos=eos, t=0.0) # Check the total velocity component normal # to each surface. It should be zero. The From 8fc775fedc5bb56f9666bc6e0807085a1909d1f7 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 11 May 2021 09:55:27 -0500 Subject: [PATCH 083/385] Update doublemach to use Navier-Stokes, update inviscid boundaries st they can be used in NS --- examples/autoignition-mpi.py | 2 +- examples/doublemach-mpi.py | 14 +++++++-- examples/nsmix-mpi.py | 2 +- mirgecom/boundary.py | 55 +++++++++++++++++++++++++++++++++++- mirgecom/initializers.py | 46 ++++++++++++++++-------------- mirgecom/navierstokes.py | 17 ++++++----- mirgecom/viscous.py | 4 +-- test/test_init.py | 4 +-- 8 files changed, 106 insertions(+), 38 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 078ebec9c..cb5c99e4b 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -175,7 +175,7 @@ def main(ctx_factory=cl.create_some_context): my_boundary = AdiabaticSlipBoundary() boundaries = {BTAG_ALL: my_boundary} - current_state = initializer(eos=eos, x_vec=nodes, t=0) + current_state = initializer(eos=eos, x_vec=nodes, time=0) # Inspection at physics debugging time if debug: diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index e58ea34c6..a1f18cf7c 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -23,6 +23,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ + import logging import pyopencl as cl import pyopencl.tools as cl_tools @@ -36,7 +37,8 @@ from grudge.shortcuts import make_visualizer -from mirgecom.euler import inviscid_operator, split_conserved +from mirgecom.navierstokes import ns_operator +from mirgecom.fluid import split_conserved from mirgecom.artificial_viscosity import ( av_operator, smoothness_indicator @@ -55,6 +57,7 @@ from mirgecom.boundary import AdiabaticSlipBoundary, PrescribedBoundary from mirgecom.initializers import DoubleMachReflection from mirgecom.eos import IdealSingleGas +from mirgecom.transport import SimpleTransport logger = logging.getLogger(__name__) @@ -122,7 +125,12 @@ def main(ctx_factory=cl.create_some_context): current_cfl = 0.1 current_dt = 1.0e-4 current_t = 0 - eos = IdealSingleGas() + # {{{ Initialize simple transport model + kappa = 1e-5 + sigma = 1e-5 + transport_model = SimpleTransport(viscosity=sigma, thermal_conductivity=kappa) + # }}} + eos = IdealSingleGas(transport_model=transport_model) initializer = DoubleMachReflection() casename = "doubleMach" @@ -195,7 +203,7 @@ def main(ctx_factory=cl.create_some_context): ) def my_rhs(t, state): - return inviscid_operator( + return ns_operator( discr, q=state, t=t, boundaries=boundaries, eos=eos ) + av_operator( discr, t=t, q=state, boundaries=boundaries, alpha=alpha, eos=eos, diff --git a/examples/nsmix-mpi.py b/examples/nsmix-mpi.py index 2ff7b888a..7797bceaa 100644 --- a/examples/nsmix-mpi.py +++ b/examples/nsmix-mpi.py @@ -193,7 +193,7 @@ def main(ctx_factory=cl.create_some_context): # my_boundary = AdiabaticSlipBoundary() my_boundary = IsothermalNoSlipBoundary(wall_temperature=can_t) visc_bnds = {BTAG_ALL: my_boundary} - current_state = initializer(eos=eos, x_vec=nodes, t=0) + current_state = initializer(eos=eos, x_vec=nodes, time=0) # Inspection at physics debugging time if debug: diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index bd681a3a9..36ae7e914 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -135,7 +135,8 @@ def __init__(self, inviscid_boundary_flux_func=None, boundary_pair_func=None, inviscid_facial_flux_func=None, fluid_solution_func=None, fluid_solution_flux_func=None, scalar_numerical_flux_func=None, fluid_solution_gradient_func=None, - fluid_solution_gradient_flux_func=None): + fluid_solution_gradient_flux_func=None, + fluid_temperature_func=None): """Initialize the PrescribedInviscidBoundary and methods.""" self._bnd_pair_func = boundary_pair_func self._inviscid_bnd_flux_func = inviscid_boundary_flux_func @@ -153,6 +154,7 @@ def __init__(self, inviscid_boundary_flux_func=None, boundary_pair_func=None, from mirgecom.flux import central_vector_flux if not self._fluid_soln_grad_flux_func: self._fluid_soln_grad_flux_func = central_vector_flux + self._fluid_temperature_func = fluid_temperature_func def _boundary_quantity(self, discr, btag, quantity, **kwargs): """Get a boundary quantity on local boundary, or projected to "all_faces".""" @@ -232,6 +234,57 @@ def s_boundary_flux(self, discr, btag, grad_q, **kwargs): **kwargs ) + def t_boundary_flux(self, discr, btag, q, eos, **kwargs): + """Get the "temperature flux" through boundary *btag*.""" + q_minus = discr.project("vol", btag, q) + cv_minus = split_conserved(discr.dim, q_minus) + t_minus = eos.temperature(cv_minus) + if self._fluid_temperature_func: + actx = q[0].array_context + boundary_discr = discr.discr_from_dd(btag) + nodes = thaw(actx, boundary_discr.nodes()) + t_plus = self._fluid_temperature_func(nodes, q=q_minus, + temperature=t_minus, eos=eos, + **kwargs) + else: + t_plus = -t_minus + # t_plus = 0*t_minus + self._wall_temp + actx = cv_minus.mass.array_context + nhat = thaw(actx, discr.normal(btag)) + bnd_tpair = TracePair(btag, interior=t_minus, exterior=t_plus) + + return self._boundary_quantity(discr, btag, + self._scalar_num_flux_func(bnd_tpair, nhat), + **kwargs) + + def viscous_boundary_flux(self, discr, btag, eos, q, grad_q, grad_t, **kwargs): + """Get the viscous part of the physical flux across the boundary *btag*.""" + q_tpair = self.boundary_pair(discr, btag=btag, q=q, eos=eos, **kwargs) + cv_minus = split_conserved(discr.dim, q_tpair.int) + + grad_q_minus = discr.project("vol", btag, grad_q) + grad_q_tpair = TracePair(btag, interior=grad_q_minus, exterior=grad_q_minus) + + t_minus = eos.temperature(cv_minus) + if self._fluid_temperature_func: + actx = q[0].array_context + boundary_discr = discr.discr_from_dd(btag) + nodes = thaw(actx, boundary_discr.nodes()) + t_plus = self._fluid_temperature_func(nodes, q=q_tpair.exterior, + temperature=t_minus, eos=eos, + **kwargs) + else: + t_plus = -t_minus + + t_tpair = TracePair(btag, interior=t_minus, exterior=t_plus) + + grad_t_minus = discr.project("vol", btag, grad_t) + grad_t_tpair = TracePair(btag, interior=grad_t_minus, exterior=grad_t_minus) + + from mirgecom.viscous import viscous_facial_flux + return viscous_facial_flux(discr, eos, q_tpair, grad_q_tpair, + t_tpair, grad_t_tpair) + class PrescribedBoundary(PrescribedInviscidBoundary): """Boundary condition prescribes boundary soln with user-specified function. diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 35a5135bb..20c72a979 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -131,7 +131,7 @@ def __init__( self._center = np.array(center) self._velocity = np.array(velocity) - def __call__(self, x_vec, *, t=0, eos=None, **kwargs): + def __call__(self, x_vec, *, time=0, eos=None, **kwargs): """ Create the isentropic vortex solution at time *t* at locations *x_vec*. @@ -141,13 +141,14 @@ def __call__(self, x_vec, *, t=0, eos=None, **kwargs): Parameters ---------- - t: float + time: float Current time at which the solution is desired. x_vec: numpy.ndarray Nodal coordinates eos: mirgecom.eos.IdealSingleGas Equation of state class to supply method for gas *gamma*. """ + t = time if eos is None: eos = IdealSingleGas() vortex_loc = self._center + t * self._velocity @@ -226,13 +227,13 @@ def __init__( if self._xdir >= self._dim: self._xdir = self._dim - 1 - def __call__(self, x_vec, *, t=0, eos=None, **kwargs): + def __call__(self, x_vec, *, eos=None, time=0, **kwargs): """ Create the 1D Sod's shock solution at locations *x_vec*. Parameters ---------- - t: float + time: float Current time at which the solution is desired (unused) x_vec: numpy.ndarray Nodal coordinates @@ -319,7 +320,7 @@ def __init__( self._shock_location = shock_location self._shock_speed = shock_speed - def __call__(self, x_vec, *, t=0, eos=None, **kwargs): + def __call__(self, x_vec, *, eos=None, time=0, **kwargs): r""" Create double mach reflection solution at locations *x_vec*. @@ -331,13 +332,14 @@ def __call__(self, x_vec, *, t=0, eos=None, **kwargs): Parameters ---------- - t: float + time: float Time at which to compute the solution x_vec: numpy.ndarray Nodal coordinates eos: :class:`mirgecom.eos.GasEOS` Equation of state class to be used in construction of soln (if needed) """ + t = time # Fail if numdim is other than 2 if(len(x_vec)) != 2: raise ValueError("Case only defined for 2 dimensions") @@ -449,7 +451,7 @@ def __init__( self._rho0 = rho0 self._rhoamp = rhoamp - def __call__(self, x_vec, *, t=0, eos=None, **kwargs): + def __call__(self, x_vec, *, eos=None, time=0, **kwargs): """ Create the lump-of-mass solution at time *t* and locations *x_vec*. @@ -458,13 +460,14 @@ def __call__(self, x_vec, *, t=0, eos=None, **kwargs): Parameters ---------- - t: float + time: float Current time at which the solution is desired x_vec: numpy.ndarray Nodal coordinates eos: :class:`mirgecom.eos.IdealSingleGas` Equation of state class with method to supply gas *gamma*. """ + t = time if eos is None: eos = IdealSingleGas() if x_vec.shape != (self._dim,): @@ -489,7 +492,7 @@ def __call__(self, x_vec, *, t=0, eos=None, **kwargs): return join_conserved(dim=self._dim, mass=mass, energy=energy, momentum=mom) - def exact_rhs(self, discr, q, t=0.0): + def exact_rhs(self, discr, q, time=0.0): """ Create the RHS for the lump-of-mass solution at time *t*, locations *x_vec*. @@ -501,9 +504,10 @@ def exact_rhs(self, discr, q, t=0.0): q State array which expects at least the canonical conserved quantities (mass, energy, momentum) for the fluid at each point. - t: float + time: float Time at which RHS is desired """ + t = time actx = q[0].array_context nodes = thaw(actx, discr.nodes()) lump_loc = self._center + t * self._velocity @@ -621,7 +625,7 @@ def __init__( self._spec_centers = spec_centers self._spec_amplitudes = spec_amplitudes - def __call__(self, x_vec, *, t=0, eos=None, **kwargs): + def __call__(self, x_vec, *, eos=None, time=0, **kwargs): """ Create a multi-component lump solution at time *t* and locations *x_vec*. @@ -631,13 +635,14 @@ def __call__(self, x_vec, *, t=0, eos=None, **kwargs): Parameters ---------- - t: float + time: float Current time at which the solution is desired x_vec: numpy.ndarray Nodal coordinates eos: :class:`mirgecom.eos.IdealSingleGas` Equation of state class with method to supply gas *gamma*. """ + t = time if eos is None: eos = IdealSingleGas() if x_vec.shape != (self._dim,): @@ -666,7 +671,7 @@ def __call__(self, x_vec, *, t=0, eos=None, **kwargs): return join_conserved(dim=self._dim, mass=mass, energy=energy, momentum=mom, species_mass=species_mass) - def exact_rhs(self, discr, q, t=0.0): + def exact_rhs(self, discr, q, time=0.0): """ Create a RHS for multi-component lump soln at time *t*, locations *x_vec*. @@ -678,9 +683,10 @@ def exact_rhs(self, discr, q, t=0.0): q State array which expects at least the canonical conserved quantities (mass, energy, momentum) for the fluid at each point. - t: float + time: float Time at which RHS is desired """ + t = time actx = q[0].array_context nodes = thaw(actx, discr.nodes()) loc_update = t * self._velocity @@ -760,8 +766,6 @@ def __call__(self, x_vec, q, eos=None, **kwargs): Parameters ---------- - t: float - Current time at which the solution is desired (unused) x_vec: numpy.ndarray Nodal coordinates eos: :class:`mirgecom.eos.GasEOS` @@ -839,13 +843,13 @@ def __init__( self._e = e self._dim = dim - def __call__(self, x_vec, *, t=0, eos=None, **kwargs): + def __call__(self, x_vec, *, eos=None, time=0, **kwargs): """ Create a uniform flow solution at locations *x_vec*. Parameters ---------- - t: float + time: float Current time at which the solution is desired (unused) x_vec: numpy.ndarray Nodal coordinates @@ -934,7 +938,7 @@ def __init__( self._temperature = temperature self._massfracs = massfractions - def __call__(self, x_vec, eos, *, t=0.0, **kwargs): + def __call__(self, x_vec, eos, *, time=0.0, **kwargs): """ Create the mixture state at locations *x_vec* (t is ignored). @@ -947,8 +951,8 @@ def __call__(self, x_vec, eos, *, t=0.0, **kwargs): these functions: `eos.get_density` `eos.get_internal_energy` - t: float - Time is ignored by this solution intitializer + time: float + Time is ignored by this solution intitializer (unused) """ if x_vec.shape != (self._dim,): raise ValueError(f"Position vector has unexpected dimensionality," diff --git a/mirgecom/navierstokes.py b/mirgecom/navierstokes.py index 3fa80f2b5..4a3969f88 100644 --- a/mirgecom/navierstokes.py +++ b/mirgecom/navierstokes.py @@ -131,7 +131,8 @@ def scalar_flux_interior(int_tpair): return discr.project(int_tpair.dd, "all_faces", flux_weak) def get_q_flux_bnd(btag): - return boundaries[btag].q_boundary_flux(discr, btag, eos, q, time=t) + return boundaries[btag].q_boundary_flux(discr, btag=btag, q=q, eos=eos, + time=t) q_int_tpair = interior_trace_pair(discr, q) q_part_pairs = cross_rank_trace_pairs(discr, q) @@ -144,7 +145,9 @@ def get_q_flux_bnd(btag): # Temperature gradient for conductive heat flux: [Ihme_2014]_ eqn (3b) # - now computed, *not* communicated def get_t_flux_bnd(btag): - return boundaries[btag].t_boundary_flux(discr, btag, eos, q, time=t) + return boundaries[btag].t_boundary_flux(discr, btag=btag, q=q, eos=eos, + time=t) + gas_t = eos.temperature(cv) t_int_tpair = TracePair("int_faces", interior=eos.temperature( @@ -162,12 +165,12 @@ def get_t_flux_bnd(btag): # inviscid parts def finv_interior_face(q_tpair): - return inviscid_facial_flux(discr, eos, q_tpair) + return inviscid_facial_flux(discr, eos=eos, q_tpair=q_tpair) # inviscid part of bcs applied here def finv_domain_boundary(btag): - return boundaries[btag].inviscid_boundary_flux(discr, btag, eos=eos, q=q, - time=t) + return boundaries[btag].inviscid_boundary_flux(discr, btag=btag, eos=eos, + q=q, time=t) # viscous parts s_int_pair = interior_trace_pair(discr, grad_q) @@ -200,8 +203,8 @@ def visc_bnd_flux(btag): # NS RHS return dg_div_low( discr, ( # volume part - viscous_flux(discr, eos, q=q, grad_q=grad_q, t=gas_t, grad_t=grad_t) - - inviscid_flux(discr, eos, q)), + viscous_flux(discr, eos=eos, q=q, grad_q=grad_q, t=gas_t, grad_t=grad_t) + - inviscid_flux(discr, eos=eos, q=q)), elbnd_flux( # viscous boundary discr, fvisc_interior_face, visc_bnd_flux, (q_int_tpair, s_int_pair, t_int_tpair, delt_int_pair), diff --git a/mirgecom/viscous.py b/mirgecom/viscous.py index 5e8f26217..2e8f04216 100644 --- a/mirgecom/viscous.py +++ b/mirgecom/viscous.py @@ -81,10 +81,10 @@ def diffusive_flux(discr, eos, q, grad_q): fractions ${Y}_{\alpha}$. """ cv = split_conserved(discr.dim, q) - grad_cv = split_conserved(discr.dim, grad_q) nspecies = len(cv.species_mass) - transport = eos.transport_model() + grad_cv = split_conserved(discr.dim, grad_q) + transport = eos.transport_model() grad_y = species_mass_fraction_gradient(discr, cv, grad_cv) d = transport.species_diffusivity(eos, cv) diff --git a/test/test_init.py b/test/test_init.py index 36c718348..e26d42cae 100644 --- a/test/test_init.py +++ b/test/test_init.py @@ -218,7 +218,7 @@ def test_shock_init(ctx_factory): nodes = thaw(actx, discr.nodes()) initr = SodShock1D() - initsoln = initr(t=0.0, x_vec=nodes) + initsoln = initr(time=0.0, x_vec=nodes) print("Sod Soln:", initsoln) xpl = 1.0 xpr = 0.1 @@ -259,7 +259,7 @@ def test_uniform(ctx_factory, dim): from mirgecom.initializers import Uniform initr = Uniform(dim=dim) - initsoln = initr(t=0.0, x_vec=nodes) + initsoln = initr(time=0.0, x_vec=nodes) tol = 1e-15 ssoln = split_conserved(dim, initsoln) From 194bd46e082bec83d6eb90bb5b36450dfd7027ce Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 11 May 2021 11:43:14 -0500 Subject: [PATCH 084/385] Tweak a bit for new interface and VTU overwrite for vortex example. --- examples/vortex-mpi.py | 2 +- test/test_euler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index f13949fdb..9ab2a0839 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -178,7 +178,7 @@ def my_checkpoint(step, t, dt, state): exact_soln=initializer, vizname=casename, step=step, t=t, dt=dt, nstatus=nstatus, nviz=nviz, exittol=exittol, constant_cfl=constant_cfl, comm=comm, - vis_timer=vis_timer) + vis_timer=vis_timer, overwrite=True) try: (current_step, current_t, current_state) = \ diff --git a/test/test_euler.py b/test/test_euler.py index ac8572696..dffd7ba91 100644 --- a/test/test_euler.py +++ b/test/test_euler.py @@ -848,7 +848,7 @@ def rhs(t, q): logger.info("Writing final dump.") maxerr = max(write_soln(False)) else: - expected_result = initializer(nodes, t=t) + expected_result = initializer(nodes, time=t) maxerr = discr.norm(fields - expected_result, np.inf) logger.info(f"Max Error: {maxerr}") From 389c22e1352d36e6c6bb8351bee62d033ff5f843 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 11 May 2021 13:24:52 -0500 Subject: [PATCH 085/385] Fix some cosmetics only. --- examples/heat-source-mpi.py | 1 - mirgecom/viscous.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/heat-source-mpi.py b/examples/heat-source-mpi.py index b683498cd..196654a61 100644 --- a/examples/heat-source-mpi.py +++ b/examples/heat-source-mpi.py @@ -44,7 +44,6 @@ @mpi_entry_point def main(): - """Drive the example.""" cl_ctx = cl.create_some_context() queue = cl.CommandQueue(cl_ctx) actx = PyOpenCLArrayContext(queue, diff --git a/mirgecom/viscous.py b/mirgecom/viscous.py index 2e8f04216..5e8f26217 100644 --- a/mirgecom/viscous.py +++ b/mirgecom/viscous.py @@ -81,10 +81,10 @@ def diffusive_flux(discr, eos, q, grad_q): fractions ${Y}_{\alpha}$. """ cv = split_conserved(discr.dim, q) - nspecies = len(cv.species_mass) - grad_cv = split_conserved(discr.dim, grad_q) + nspecies = len(cv.species_mass) transport = eos.transport_model() + grad_y = species_mass_fraction_gradient(discr, cv, grad_cv) d = transport.species_diffusivity(eos, cv) From 657aec60b10688daf520558778f625f35764befa Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Mon, 17 May 2021 13:10:10 -0500 Subject: [PATCH 086/385] Fix passing of time variable to boundary (#342) --- examples/doublemach-mpi.py | 2 +- mirgecom/euler.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 771d98a15..901bbadd1 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -207,7 +207,7 @@ def my_rhs(t, state): discr, q=state, t=t, boundaries=boundaries, eos=eos ) + av_operator( discr, q=state, boundaries=boundaries, - boundary_kwargs={"t": t, "eos": eos}, alpha=alpha, + boundary_kwargs={"time": t, "eos": eos}, alpha=alpha, s0=s0, kappa=kappa ) diff --git a/mirgecom/euler.py b/mirgecom/euler.py index 0bef5679f..bab4daa16 100644 --- a/mirgecom/euler.py +++ b/mirgecom/euler.py @@ -108,7 +108,8 @@ def euler_operator(discr, eos, boundaries, q, t=0.0): inviscid_facial_flux(discr, eos=eos, q_tpair=interior_trace_pair(discr, q)) + sum(inviscid_facial_flux(discr, eos=eos, q_tpair=part_tpair) for part_tpair in cross_rank_trace_pairs(discr, q)) - + sum(boundaries[btag].inviscid_boundary_flux(discr, btag=btag, q=q, eos=eos) + + sum(boundaries[btag].inviscid_boundary_flux(discr, btag=btag, q=q, eos=eos, + time=t) for btag in boundaries) ) From 6fc2200c59399fb8a809055ecffd25191879b916 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 19 May 2021 11:20:50 -0500 Subject: [PATCH 087/385] Add AV interface routine to Moving Noslip. --- mirgecom/boundary.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index bd7ae4f04..d3579458d 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -419,6 +419,7 @@ class AdiabaticNoslipMovingBoundary(PrescribedInviscidBoundary): .. automethod:: adiabatic_noslip_pair .. automethod:: exterior_soln + .. automethod:: exterior_grad_q """ def __init__(self, wall_velocity=None, dim=2): @@ -460,6 +461,10 @@ def exterior_soln(self, discr, q, btag, **kwargs): return bndry_soln + def exterior_grad_q(self, nodes, nhat, grad_q, **kwargs): + """Get the exterior solution on the boundary.""" + return(-grad_q) + class IsothermalNoSlipBoundary(FluidBC): r"""Isothermal no-slip viscous wall boundary. From 3432f8a45fb87530b598852a23c03a2d20ba995e Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 21 May 2021 07:53:32 -0500 Subject: [PATCH 088/385] Update initializers.py --- mirgecom/initializers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index f11278666..1209c20ce 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -1005,8 +1005,9 @@ def __init__( specifies the number of dimensions for the solution normal_dir: int specifies the direction (plane) the discontinuity is applied in - disc_location: float or Function[float] - location of discontinuity (in time) + disc_location: float or Callable + fixed location of discontinuity or optionally a function that + returns the time-dependent location. nspecies: int specifies the number of mixture species pressure_left: float From 839fef4aaedd0ecc6de85479a0da65753686c02d Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 21 May 2021 07:58:04 -0500 Subject: [PATCH 089/385] Placate flake8 --- mirgecom/initializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 1209c20ce..85e81eb7b 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -1006,7 +1006,7 @@ def __init__( normal_dir: int specifies the direction (plane) the discontinuity is applied in disc_location: float or Callable - fixed location of discontinuity or optionally a function that + fixed location of discontinuity or optionally a function that returns the time-dependent location. nspecies: int specifies the number of mixture species From 347c61db1cdb486974e7547ea107615ebcb27f9f Mon Sep 17 00:00:00 2001 From: w-hagen <26756513+w-hagen@users.noreply.github.com> Date: Sat, 22 May 2021 00:27:19 -0500 Subject: [PATCH 090/385] Fix loss of bc grad function in inits (#352) --- mirgecom/boundary.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 5d9b59508..8737e5a6e 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -353,7 +353,8 @@ class AdiabaticSlipBoundary(PrescribedInviscidBoundary): def __init__(self): """Initialize AdiabaticSlipBoundary.""" PrescribedInviscidBoundary.__init__( - self, boundary_pair_func=self.adiabatic_slip_pair + self, boundary_pair_func=self.adiabatic_slip_pair, + fluid_solution_gradient_func=self.exterior_grad_q ) def adiabatic_slip_pair(self, discr, q, btag, **kwargs): @@ -425,7 +426,8 @@ class AdiabaticNoslipMovingBoundary(PrescribedInviscidBoundary): def __init__(self, wall_velocity=None, dim=2): """Initialize boundary device.""" PrescribedInviscidBoundary.__init__( - self, boundary_pair_func=self.adiabatic_noslip_pair + self, boundary_pair_func=self.adiabatic_noslip_pair, + fluid_solution_gradient_func=self.exterior_grad_q ) # Check wall_velocity (assumes dim is correct) if wall_velocity is None: From 9226ef48d673a65c77372f67962b3cceb9a31a65 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sun, 23 May 2021 13:11:09 -0500 Subject: [PATCH 091/385] Merge down Wyatts BC catch #352 --- mirgecom/boundary.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 5d9b59508..8737e5a6e 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -353,7 +353,8 @@ class AdiabaticSlipBoundary(PrescribedInviscidBoundary): def __init__(self): """Initialize AdiabaticSlipBoundary.""" PrescribedInviscidBoundary.__init__( - self, boundary_pair_func=self.adiabatic_slip_pair + self, boundary_pair_func=self.adiabatic_slip_pair, + fluid_solution_gradient_func=self.exterior_grad_q ) def adiabatic_slip_pair(self, discr, q, btag, **kwargs): @@ -425,7 +426,8 @@ class AdiabaticNoslipMovingBoundary(PrescribedInviscidBoundary): def __init__(self, wall_velocity=None, dim=2): """Initialize boundary device.""" PrescribedInviscidBoundary.__init__( - self, boundary_pair_func=self.adiabatic_noslip_pair + self, boundary_pair_func=self.adiabatic_noslip_pair, + fluid_solution_gradient_func=self.exterior_grad_q ) # Check wall_velocity (assumes dim is correct) if wall_velocity is None: From e7df3d952a75634c629426098cca0d959b36173a Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Sun, 23 May 2021 14:44:35 -0500 Subject: [PATCH 092/385] Install PyYAML as a pip package. --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index b60e76ad2..b0654c66a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ pymetis importlib-resources psutil gmsh +PyYAML # The following packages will be git cloned by emirge: --editable git+https://github.com/inducer/pymbolic.git#egg=pymbolic From d31875bccfa130948d9b6705bb2a13ea2014e7e5 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sun, 23 May 2021 15:26:43 -0500 Subject: [PATCH 093/385] Add YAML file reader cc(@anderson2981) --- mirgecom/io.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mirgecom/io.py b/mirgecom/io.py index a894c8ea5..d2b35f6b2 100644 --- a/mirgecom/io.py +++ b/mirgecom/io.py @@ -3,6 +3,7 @@ .. autofunction:: make_status_message .. autofunction:: make_rank_fname .. autofunction:: make_par_fname +.. autofunction:: read_and_distribute_yaml_data """ __copyright__ = """ @@ -74,3 +75,16 @@ def make_rank_fname(basename, rank=0, step=0, t=0): def make_par_fname(basename, step=0, t=0): r"""Make parallel visualization filename.""" return f"{basename}-{step:06d}.pvtu" + + +def read_and_distribute_yaml_data(mpi_comm, file_path): + """Read a YAML file on one rank, broadcast result to world.""" + import yaml + rank = mpi_comm.Get_rank() + if rank == 0: + with open(file_path) as f: + input_data = yaml.load(f, Loader=yaml.FullLoader) + else: + input_data = None + mpi_comm.bcast(input_data, root=0) + return input_data From 2cc8a7f50f33bf272540614870bf0ed531bcd693 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Thu, 27 May 2021 07:09:54 -0700 Subject: [PATCH 094/385] Add parameters stuff --- mirgecom/simutil.py | 23 ++++++++++++++++ test/test_util.py | 66 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 test/test_util.py diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index fe7f4feb4..ee68cfcf4 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -41,6 +41,29 @@ logger = logging.getLogger(__name__) +class MIRGEComParameters: + """Simple parameters object.""" + def __init__(self, **kwargs): + self._parameters = kwargs + + @property + def parameters(self): + """Grab the parameters.""" + return self._parameters + + def update(self, **kwargs): + """Update parameters with new or replacement parameters or values.""" + self._parameters.update(kwargs) + + def read(self, file_path): + """Read new or replacement values from a file at system path *file_path*.""" + import importlib.util + spec = importlib.util.spec_from_file_location("user_parameters", file_path) + foo = importlib.util.module_from_spec(spec) + spec.loader.exec_module(foo) + self._parameters.update(foo.mirgecom_parameters.parameters) + + def check_step(step, interval): """ Check step number against a user-specified interval. diff --git a/test/test_util.py b/test/test_util.py new file mode 100644 index 000000000..b77cab475 --- /dev/null +++ b/test/test_util.py @@ -0,0 +1,66 @@ +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import logging +from mirgecom.simutil import MIRGEComParameters + +logger = logging.getLogger(__name__) + + +def test_mirgecom_parameters(): + """Quick test of MIRGE-Com parameters container.""" + test_params = MIRGEComParameters(dim=2, order=3, casename="hello") + my_params = test_params.parameters + print(f"{test_params.parameters}") + assert len(my_params) == 3 + assert my_params["dim"] == 2 + assert my_params["order"] == 3 + assert my_params["casename"] == "hello" + + test_params.update(order=4, casename="goodbye", hello="hello") + my_params = test_params.parameters + assert len(my_params) == 4 + assert my_params["order"] == 4 + assert my_params["dim"] == 2 + assert my_params["casename"] == "goodbye" + assert my_params["hello"] == "hello" + + params_string = ( + f"from mirgecom.simutil import MIRGEComParameters" + f"\nmirgecom_parameters = MIRGEComParameters(" + f"\ndim=5, newparam=\"string\")" + ) + + file1 = open("test_params_fjsfjksd.py", "a") + file1.write(params_string) + file1.close() + test_params.read("test_params_fjsfjksd.py") + my_params = test_params.parameters + assert len(my_params) == 5 + assert my_params["dim"] == 5 + assert my_params["newparam"] == "string" + import os + os.remove("test_params_fjsfjksd.py") + + From 7be62ebfa779aaa70930736a127082292ec610b2 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 28 May 2021 11:45:29 -0500 Subject: [PATCH 095/385] Re-enable timestep and CFL routines after new world grudge provided geometrical helpers. --- mirgecom/inviscid.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 4b1dba945..9c5986524 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -82,21 +82,26 @@ def get_inviscid_timestep(discr, eos, cfl, q): Currently, it's a hack waiting for the geometric_factor helpers port from grudge. """ + from mpi4py import MPI dim = discr.dim - mesh = discr.mesh order = max([grp.order for grp in discr.discr_from_dd("vol").groups]) - nelements = mesh.nelements - nel_1d = nelements ** (1.0 / (1.0 * dim)) - - # This roughly reproduces the timestep AK used in wave toy - dt = (1.0 - 0.25 * (dim - 1)) / (nel_1d * order ** 2) - return cfl * dt - -# dt_ngf = dt_non_geometric_factor(discr.mesh) -# dt_gf = dt_geometric_factor(discr.mesh) -# wavespeeds = compute_wavespeed(w,eos=eos) -# max_v = clmath.max(wavespeeds) -# return c*dt_ngf*dt_gf/max_v + cv = split_conserved(dim, q) + + import grudge.op as op + h_min_local = op.h_min_from_volume(discr) / (order * order) + from mirgecom.fluid import compute_wavespeed + local_wavespeeds = compute_wavespeed(dim, eos, cv) + max_wavespeed_local = op.nodal_max(discr, "vol", local_wavespeeds) + + mpi_comm = discr.get_comm() + if mpi_comm is None: + max_wavespeed_global = max_wavespeed_local + h_min_global = h_min_local + else: + h_min_global = mpi_comm.allreduce(h_min_local, op=MPI.MIN) + max_wavespeed_global = mpi_comm.allreduce(max_wavespeed_local, op=MPI.MAX) + + return cfl * h_min_global / max_wavespeed_global def get_inviscid_cfl(discr, eos, dt, q): From 7531492b61a086f570ff92a95a17b06038d8decd Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 28 May 2021 11:47:39 -0500 Subject: [PATCH 096/385] Use mpi comm from discr. --- mirgecom/inviscid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 9c5986524..bf8bbce57 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -93,7 +93,7 @@ def get_inviscid_timestep(discr, eos, cfl, q): local_wavespeeds = compute_wavespeed(dim, eos, cv) max_wavespeed_local = op.nodal_max(discr, "vol", local_wavespeeds) - mpi_comm = discr.get_comm() + mpi_comm = discr.mpi_communicator if mpi_comm is None: max_wavespeed_global = max_wavespeed_local h_min_global = h_min_local From 3d9e74267c9f6ef22cc9fa3efac25173e268c053 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 28 May 2021 11:57:08 -0500 Subject: [PATCH 097/385] Apply simplication from (live) @thomasgibson review. --- mirgecom/inviscid.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index bf8bbce57..3bef6fd2c 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -95,11 +95,10 @@ def get_inviscid_timestep(discr, eos, cfl, q): mpi_comm = discr.mpi_communicator if mpi_comm is None: - max_wavespeed_global = max_wavespeed_local - h_min_global = h_min_local - else: - h_min_global = mpi_comm.allreduce(h_min_local, op=MPI.MIN) - max_wavespeed_global = mpi_comm.allreduce(max_wavespeed_local, op=MPI.MAX) + return cfl * h_min_local / max_wavespeed_local + + h_min_global = mpi_comm.allreduce(h_min_local, op=MPI.MIN) + max_wavespeed_global = mpi_comm.allreduce(max_wavespeed_local, op=MPI.MAX) return cfl * h_min_global / max_wavespeed_global From e729ebd2667123abf1fa1663cb73ef1753e35ae4 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 1 Jun 2021 07:45:18 -0500 Subject: [PATCH 098/385] Fix up params imp and test (soon to be removed) --- mirgecom/simutil.py | 2 ++ test/test_util.py | 8 +++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index ee68cfcf4..3b6f7c5d8 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -43,7 +43,9 @@ class MIRGEComParameters: """Simple parameters object.""" + def __init__(self, **kwargs): + """Initialize parameters object.""" self._parameters = kwargs @property diff --git a/test/test_util.py b/test/test_util.py index b77cab475..822c3cd7e 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -47,9 +47,9 @@ def test_mirgecom_parameters(): assert my_params["hello"] == "hello" params_string = ( - f"from mirgecom.simutil import MIRGEComParameters" - f"\nmirgecom_parameters = MIRGEComParameters(" - f"\ndim=5, newparam=\"string\")" + "from mirgecom.simutil import MIRGEComParameters" + "\nmirgecom_parameters = MIRGEComParameters(" + "\ndim=5, newparam='string')" ) file1 = open("test_params_fjsfjksd.py", "a") @@ -62,5 +62,3 @@ def test_mirgecom_parameters(): assert my_params["newparam"] == "string" import os os.remove("test_params_fjsfjksd.py") - - From f1dbc3e63d2b6387acfb0d5f2a465e22fcc776f6 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 2 Jun 2021 16:01:26 -0500 Subject: [PATCH 099/385] Check for CV type. --- mirgecom/artificial_viscosity.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 0bb1b7d10..3c13a4e6e 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -213,6 +213,8 @@ def av_operator(discr, boundaries, q, alpha, boundary_kwargs=None, **kwargs): for btag, bnd in boundaries.items()) if isinstance(q, np.ndarray): q_bnd_flux2 = np.stack(q_bnd_flux2) + if isinstance(q_bnd_flux2, ConservedVars): + q_bnd_flux2 = q_bnd_flux2.join() q_bnd_flux = q_bnd_flux + q_bnd_flux2 # Compute R From 5fac3cb3d47213ab6e1b3dad2f9cdaaec522ebd0 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 3 Jun 2021 11:31:21 -0500 Subject: [PATCH 100/385] Use grudge dt finding utils (#371) --- examples/vortex-mpi.py | 7 ++++++- mirgecom/inviscid.py | 26 +++++++++++++++++--------- requirements.txt | 2 +- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index bbe9035ea..0f4c20d02 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -52,6 +52,7 @@ from mirgecom.boundary import PrescribedBoundary from mirgecom.initializers import Vortex2D from mirgecom.eos import IdealSingleGas +from mirgecom.inviscid import get_inviscid_cfl from logpyle import IntervalTimer from mirgecom.euler import extract_vars_for_logging, units_for_logging @@ -172,11 +173,15 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): + cfl = get_inviscid_cfl(discr, eos=eos, dt=current_dt, q=state) + viz_fields = [ + ("cfl", cfl) + ] return sim_checkpoint(discr, visualizer, eos, q=state, exact_soln=initializer, vizname=casename, step=step, t=t, dt=dt, nstatus=nstatus, nviz=nviz, exittol=exittol, constant_cfl=constant_cfl, comm=comm, - vis_timer=vis_timer) + vis_timer=vis_timer, viz_fields=viz_fields) try: (current_step, current_t, current_state) = \ diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 3bef6fd2c..07642bba6 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -84,23 +84,31 @@ def get_inviscid_timestep(discr, eos, cfl, q): """ from mpi4py import MPI dim = discr.dim - order = max([grp.order for grp in discr.discr_from_dd("vol").groups]) cv = split_conserved(dim, q) - import grudge.op as op - h_min_local = op.h_min_from_volume(discr) / (order * order) + from grudge.dt_utils import (dt_non_geometric_factor, + dt_geometric_factor) + + dt_factor = ( + dt_non_geometric_factor(discr) * dt_geometric_factor(discr) + ) + + from grudge.op import nodal_min from mirgecom.fluid import compute_wavespeed - local_wavespeeds = compute_wavespeed(dim, eos, cv) - max_wavespeed_local = op.nodal_max(discr, "vol", local_wavespeeds) + cell_dts = dt_factor / compute_wavespeed(dim, eos, cv) + dt_min_local = nodal_min(discr, "vol", cell_dts) mpi_comm = discr.mpi_communicator if mpi_comm is None: - return cfl * h_min_local / max_wavespeed_local + dt_min_global = dt_min_local + else: + dt_min_global = mpi_comm.allreduce(dt_min_local, op=MPI.MIN) - h_min_global = mpi_comm.allreduce(h_min_local, op=MPI.MIN) - max_wavespeed_global = mpi_comm.allreduce(max_wavespeed_local, op=MPI.MAX) + # this routine is collective - so this error should be ok + # if dt_min_global < 0: + # raise ValueError("Negative timstep detected.") - return cfl * h_min_global / max_wavespeed_global + return cfl * dt_min_global def get_inviscid_cfl(discr, eos, dt, q): diff --git a/requirements.txt b/requirements.txt index 5a5d4b277..f8210f3c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ psutil --editable git+https://github.com/inducer/modepy.git#egg=modepy --editable git+https://github.com/inducer/arraycontext.git#egg=arraycontext --editable git+https://github.com/inducer/meshmode.git#egg=meshmode ---editable git+https://github.com/inducer/grudge.git#egg=grudge +--editable git+https://github.com/inducer/grudge.git@dt_utils#egg=grudge --editable git+https://github.com/inducer/pytato.git#egg=pytato --editable git+https://github.com/ecisneros8/pyrometheus.git#egg=pyrometheus --editable git+https://github.com/illinois-ceesd/logpyle.git#egg=logpyle From 711995a76927ba68f773c4ad599f772750298dbc Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 3 Jun 2021 12:54:14 -0500 Subject: [PATCH 101/385] Correct for built-in grudge geofacs, local cfl viz in selected examples. --- examples/autoignition-mpi.py | 7 +++++-- examples/lump-mpi.py | 7 ++++++- examples/vortex-mpi.py | 4 ++-- mirgecom/inviscid.py | 22 ++++------------------ mirgecom/simutil.py | 11 +++++++++-- 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 078ebec9c..95ae662c7 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -52,7 +52,8 @@ from mirgecom.boundary import AdiabaticSlipBoundary from mirgecom.initializers import MixtureInitializer from mirgecom.eos import PyrometheusMixture -from mirgecom.euler import split_conserved +from mirgecom.fluid import split_conserved +from mirgecom.inviscid import get_inviscid_cfl import cantera import pyrometheus as pyro @@ -228,7 +229,9 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): cv = split_conserved(dim, state) reaction_rates = eos.get_production_rates(cv) - viz_fields = [("reaction_rates", reaction_rates)] + local_cfl = get_inviscid_cfl(discr, eos=eos, dt=current_dt, q=state) + viz_fields = [("reaction_rates", reaction_rates), + ("cfl", local_cfl)] return sim_checkpoint(discr, visualizer, eos, q=state, vizname=casename, step=step, t=t, dt=dt, nstatus=nstatus, nviz=nviz, diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 75d2fdb01..a9d70ee67 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -51,6 +51,7 @@ from mirgecom.boundary import PrescribedBoundary from mirgecom.initializers import Lump from mirgecom.eos import IdealSingleGas +from mirgecom.inviscid import get_inviscid_cfl logger = logging.getLogger(__name__) @@ -127,7 +128,11 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - return sim_checkpoint(discr, visualizer, eos, q=state, + local_cfl = get_inviscid_cfl(discr, eos=eos, dt=current_dt, q=state) + viz_fields = [ + ("cfl", local_cfl) + ] + return sim_checkpoint(discr, visualizer, eos, q=state, viz_fields=viz_fields, exact_soln=initializer, vizname=casename, step=step, t=t, dt=dt, nstatus=nstatus, nviz=nviz, exittol=exittol, constant_cfl=constant_cfl, comm=comm) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 0f4c20d02..ac07a5f44 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -173,9 +173,9 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - cfl = get_inviscid_cfl(discr, eos=eos, dt=current_dt, q=state) + local_cfl = get_inviscid_cfl(discr, eos=eos, dt=current_dt, q=state) viz_fields = [ - ("cfl", cfl) + ("cfl", local_cfl) ] return sim_checkpoint(discr, visualizer, eos, q=state, exact_soln=initializer, vizname=casename, step=step, diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 07642bba6..c4ff5ae3d 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -77,12 +77,11 @@ def inviscid_flux(discr, eos, q): def get_inviscid_timestep(discr, eos, cfl, q): - """Routine (will) return the (local) maximum stable inviscid timestep. + """Routine returns the cell-local maximum stable inviscid timestep. Currently, it's a hack waiting for the geometric_factor helpers port from grudge. """ - from mpi4py import MPI dim = discr.dim cv = split_conserved(dim, q) @@ -93,25 +92,12 @@ def get_inviscid_timestep(discr, eos, cfl, q): dt_non_geometric_factor(discr) * dt_geometric_factor(discr) ) - from grudge.op import nodal_min from mirgecom.fluid import compute_wavespeed - cell_dts = dt_factor / compute_wavespeed(dim, eos, cv) - dt_min_local = nodal_min(discr, "vol", cell_dts) + inviscid_dt = cfl * dt_factor / compute_wavespeed(dim, eos, cv) - mpi_comm = discr.mpi_communicator - if mpi_comm is None: - dt_min_global = dt_min_local - else: - dt_min_global = mpi_comm.allreduce(dt_min_local, op=MPI.MIN) - - # this routine is collective - so this error should be ok - # if dt_min_global < 0: - # raise ValueError("Negative timstep detected.") - - return cfl * dt_min_global + return inviscid_dt def get_inviscid_cfl(discr, eos, dt, q): """Calculate and return CFL based on current state and timestep.""" - wanted_dt = get_inviscid_timestep(discr, eos=eos, cfl=1.0, q=q) - return dt / wanted_dt + return dt / get_inviscid_timestep(discr, eos=eos, cfl=1.0, q=1) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index bdc72d653..29cae8dfe 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -66,9 +66,16 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, t_final, constant_cfl=False): """Return the maximum stable dt.""" mydt = dt + dt_left = t_final - t + if dt_left < 0: + return 0.0 if constant_cfl is True: - mydt = get_inviscid_timestep(discr=discr, q=state, - cfl=cfl, eos=eos) + from grudge.op import nodal_min + mydt = nodal_min( + discr, "vol", + get_inviscid_timestep(discr=discr, q=state, + cfl=cfl, eos=eos) + ) if (t + mydt) > t_final: mydt = t_final - t return mydt From bf0dcb693f39190bafe1cf3ec4f137bdfb50fde6 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 3 Jun 2021 13:18:03 -0500 Subject: [PATCH 102/385] Report CFL in status msg. --- examples/autoignition-mpi.py | 4 +++- examples/lump-mpi.py | 4 +++- examples/vortex-mpi.py | 4 +++- mirgecom/inviscid.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 95ae662c7..d7eb1124b 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -232,8 +232,10 @@ def my_checkpoint(step, t, dt, state): local_cfl = get_inviscid_cfl(discr, eos=eos, dt=current_dt, q=state) viz_fields = [("reaction_rates", reaction_rates), ("cfl", local_cfl)] + from grudge.op import nodal_max + max_cfl = nodal_max(discr, "vol", local_cfl) return sim_checkpoint(discr, visualizer, eos, q=state, - vizname=casename, step=step, + vizname=casename, step=step, cfl=max_cfl, t=t, dt=dt, nstatus=nstatus, nviz=nviz, constant_cfl=constant_cfl, comm=comm, viz_fields=viz_fields) diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index a9d70ee67..3b568adea 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -132,9 +132,11 @@ def my_checkpoint(step, t, dt, state): viz_fields = [ ("cfl", local_cfl) ] + from grudge.op import nodal_max + max_cfl = nodal_max(discr, "vol", local_cfl) return sim_checkpoint(discr, visualizer, eos, q=state, viz_fields=viz_fields, exact_soln=initializer, vizname=casename, step=step, - t=t, dt=dt, nstatus=nstatus, nviz=nviz, + t=t, dt=dt, nstatus=nstatus, nviz=nviz, cfl=max_cfl, exittol=exittol, constant_cfl=constant_cfl, comm=comm) try: diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index ac07a5f44..ef9af8e2c 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -177,7 +177,9 @@ def my_checkpoint(step, t, dt, state): viz_fields = [ ("cfl", local_cfl) ] - return sim_checkpoint(discr, visualizer, eos, q=state, + from grudge.op import nodal_max + max_cfl = nodal_max(discr, "vol", local_cfl) + return sim_checkpoint(discr, visualizer, eos, q=state, cfl=max_cfl, exact_soln=initializer, vizname=casename, step=step, t=t, dt=dt, nstatus=nstatus, nviz=nviz, exittol=exittol, constant_cfl=constant_cfl, comm=comm, diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index c4ff5ae3d..3ef65b895 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -100,4 +100,4 @@ def get_inviscid_timestep(discr, eos, cfl, q): def get_inviscid_cfl(discr, eos, dt, q): """Calculate and return CFL based on current state and timestep.""" - return dt / get_inviscid_timestep(discr, eos=eos, cfl=1.0, q=1) + return dt / get_inviscid_timestep(discr, eos=eos, cfl=1.0, q=q) From f2f4777a9edf820453fd609df91bb9cd5a2056c0 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 3 Jun 2021 23:06:13 -0500 Subject: [PATCH 103/385] Update to new dt_geometric_factors interface. --- mirgecom/inviscid.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 3ef65b895..daa2d7406 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -77,23 +77,19 @@ def inviscid_flux(discr, eos, q): def get_inviscid_timestep(discr, eos, cfl, q): - """Routine returns the cell-local maximum stable inviscid timestep. - - Currently, it's a hack waiting for the geometric_factor helpers port - from grudge. - """ + """Routine returns the cell-local maximum stable inviscid timestep.""" dim = discr.dim cv = split_conserved(dim, q) from grudge.dt_utils import (dt_non_geometric_factor, - dt_geometric_factor) + dt_geometric_factors) - dt_factor = ( - dt_non_geometric_factor(discr) * dt_geometric_factor(discr) + dt_factors = ( + dt_non_geometric_factor(discr) * dt_geometric_factors(discr) ) from mirgecom.fluid import compute_wavespeed - inviscid_dt = cfl * dt_factor / compute_wavespeed(dim, eos, cv) + inviscid_dt = cfl * dt_factors / compute_wavespeed(dim, eos, cv) return inviscid_dt From 00e55f5bb1bd4b0f2712d1ab44c61cc772f0a4f5 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 3 Jun 2021 23:13:39 -0500 Subject: [PATCH 104/385] Clean up the inviscid dt routines a bit, evict reductions. --- mirgecom/inviscid.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index daa2d7406..09a908ebc 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -83,15 +83,12 @@ def get_inviscid_timestep(discr, eos, cfl, q): from grudge.dt_utils import (dt_non_geometric_factor, dt_geometric_factors) - - dt_factors = ( - dt_non_geometric_factor(discr) * dt_geometric_factors(discr) - ) - from mirgecom.fluid import compute_wavespeed - inviscid_dt = cfl * dt_factors / compute_wavespeed(dim, eos, cv) - return inviscid_dt + return ( + cfl * dt_non_geometric_factor(discr) * dt_geometric_factors(discr) + / compute_wavespeed(dim, eos, cv) + ) def get_inviscid_cfl(discr, eos, dt, q): From 2a3f29937cc2d6edc8ac68c628b6077842ad030e Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 3 Jun 2021 23:20:38 -0500 Subject: [PATCH 105/385] Clean up the sim util dt routine a bit, add reductions. --- mirgecom/simutil.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 29cae8dfe..5aa2a3971 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -76,9 +76,7 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, get_inviscid_timestep(discr=discr, q=state, cfl=cfl, eos=eos) ) - if (t + mydt) > t_final: - mydt = t_final - t - return mydt + return min(mydt, dt_left) class ExactSolutionMismatch(Exception): From 1b3e3d8000d5514fc313bb838f35c14b8d31ae2a Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Sat, 5 Jun 2021 00:30:24 -0500 Subject: [PATCH 106/385] Switch back to grudge main. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f8210f3c5..5a5d4b277 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ psutil --editable git+https://github.com/inducer/modepy.git#egg=modepy --editable git+https://github.com/inducer/arraycontext.git#egg=arraycontext --editable git+https://github.com/inducer/meshmode.git#egg=meshmode ---editable git+https://github.com/inducer/grudge.git@dt_utils#egg=grudge +--editable git+https://github.com/inducer/grudge.git#egg=grudge --editable git+https://github.com/inducer/pytato.git#egg=pytato --editable git+https://github.com/ecisneros8/pyrometheus.git#egg=pyrometheus --editable git+https://github.com/illinois-ceesd/logpyle.git#egg=logpyle From 80d8087aa2a5602ca14c65722d13cfbba2c550b4 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 5 Jun 2021 19:25:33 -0500 Subject: [PATCH 107/385] Sharpen the documentation, and the interface. --- mirgecom/inviscid.py | 58 ++++++++++++++++++++++++++++++++++++++------ mirgecom/simutil.py | 5 ++-- test/test_euler.py | 4 +-- 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 09a908ebc..69a85ecc1 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -76,21 +76,63 @@ def inviscid_flux(discr, eos, q): (mom / cv.mass) * cv.species_mass.reshape(-1, 1))) -def get_inviscid_timestep(discr, eos, cfl, q): - """Routine returns the cell-local maximum stable inviscid timestep.""" - dim = discr.dim - cv = split_conserved(dim, q) - +def get_inviscid_timestep(discr, eos, q): + """Routine returns the node-local maximum stable inviscid timestep. + + Parameters + ---------- + discr: grudge.eager.EagerDGDiscretization + the discretization to use + eos: mirgecom.eos.GasEOS + Implementing the pressure and temperature functions for + returning pressure and temperature as a function of the state q. + q + State array which expects at least the canonical conserved quantities + (mass, energy, momentum) for the fluid at each point. For multi-component + fluids, the conserved quantities should include + (mass, energy, momentum, species_mass), where *species_mass* is a vector + of species masses. + + Returns + ------- + class:`~grudge.dof_array.DOFArray` + The maximum stable timestep at each node. + """ from grudge.dt_utils import (dt_non_geometric_factor, dt_geometric_factors) from mirgecom.fluid import compute_wavespeed + dim = discr.dim + cv = split_conserved(dim, q) + return ( - cfl * dt_non_geometric_factor(discr) * dt_geometric_factors(discr) + dt_non_geometric_factor(discr) * dt_geometric_factors(discr) / compute_wavespeed(dim, eos, cv) ) def get_inviscid_cfl(discr, eos, dt, q): - """Calculate and return CFL based on current state and timestep.""" - return dt / get_inviscid_timestep(discr, eos=eos, cfl=1.0, q=q) + """Calculate and return node-local CFL based on current state and timestep. + + Parameters + ---------- + discr: grudge.eager.EagerDGDiscretization + the discretization to use + eos: mirgecom.eos.GasEOS + Implementing the pressure and temperature functions for + returning pressure and temperature as a function of the state q. + dt: float or class:`~grudge.dof_array.DOFArray` + A constant scalar dt or node-local dt + q + State array which expects at least the canonical conserved quantities + (mass, energy, momentum) for the fluid at each point. For multi-component + fluids, the conserved quantities should include + (mass, energy, momentum, species_mass), where *species_mass* is a vector + of species masses. + + Returns + ------- + class:`grudge.dof_array.DOFArray` + The CFL at each node. + """ + return dt / get_inviscid_timestep(discr, eos=eos, q=q) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 5aa2a3971..ab9f5e8c9 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -71,10 +71,9 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, return 0.0 if constant_cfl is True: from grudge.op import nodal_min - mydt = nodal_min( + mydt = cfl * nodal_min( discr, "vol", - get_inviscid_timestep(discr=discr, q=state, - cfl=cfl, eos=eos) + get_inviscid_timestep(discr=discr, eos=eos, q=state) ) return min(mydt, dt_left) diff --git a/test/test_euler.py b/test/test_euler.py index 5211fdd2c..7144cc802 100644 --- a/test/test_euler.py +++ b/test/test_euler.py @@ -760,7 +760,7 @@ def _euler_flow_stepper(actx, parameters): discr = EagerDGDiscretization(actx, mesh, order=order) nodes = thaw(actx, discr.nodes()) fields = initializer(nodes) - sdt = get_inviscid_timestep(discr, eos=eos, cfl=cfl, q=fields) + sdt = cfl * get_inviscid_timestep(discr, eos=eos, q=fields) initname = initializer.__class__.__name__ eosname = eos.__class__.__name__ @@ -840,7 +840,7 @@ def rhs(t, q): t += dt istep += 1 - sdt = get_inviscid_timestep(discr, eos=eos, cfl=cfl, q=fields) + sdt = cfl * get_inviscid_timestep(discr, eos=eos, q=fields) if nstepstatus > 0: logger.info("Writing final dump.") From 5377759a42364f80c1fda60a2d6753bb0ee4bd69 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 5 Jun 2021 19:46:19 -0500 Subject: [PATCH 108/385] Documutation --- mirgecom/inviscid.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 69a85ecc1..d004a4eb8 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -86,7 +86,7 @@ def get_inviscid_timestep(discr, eos, q): eos: mirgecom.eos.GasEOS Implementing the pressure and temperature functions for returning pressure and temperature as a function of the state q. - q + q: numpy.ndarray State array which expects at least the canonical conserved quantities (mass, energy, momentum) for the fluid at each point. For multi-component fluids, the conserved quantities should include @@ -95,7 +95,7 @@ def get_inviscid_timestep(discr, eos, q): Returns ------- - class:`~grudge.dof_array.DOFArray` + class:`~meshmode.dof_array.DOFArray` The maximum stable timestep at each node. """ from grudge.dt_utils import (dt_non_geometric_factor, @@ -116,14 +116,14 @@ def get_inviscid_cfl(discr, eos, dt, q): Parameters ---------- - discr: grudge.eager.EagerDGDiscretization + discr: :class:`grudge.eager.EagerDGDiscretization` the discretization to use eos: mirgecom.eos.GasEOS Implementing the pressure and temperature functions for returning pressure and temperature as a function of the state q. - dt: float or class:`~grudge.dof_array.DOFArray` + dt: float or :class:`~meshmode.dof_array.DOFArray` A constant scalar dt or node-local dt - q + q: numpy.ndarray State array which expects at least the canonical conserved quantities (mass, energy, momentum) for the fluid at each point. For multi-component fluids, the conserved quantities should include @@ -132,7 +132,7 @@ def get_inviscid_cfl(discr, eos, dt, q): Returns ------- - class:`grudge.dof_array.DOFArray` + :class:`meshmode.dof_array.DOFArray` The CFL at each node. """ return dt / get_inviscid_timestep(discr, eos=eos, q=q) From 856821c58a74b5feedcae0f06fec5aa2ea32295e Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Fri, 28 May 2021 13:45:07 -0500 Subject: [PATCH 109/385] Add pre/post callback function args to advance_state --- mirgecom/steppers.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 95e605dce..e8074b9db 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -32,8 +32,11 @@ from mirgecom.logging_quantities import set_sim_state -def _advance_state_stepper_func(rhs, timestepper, checkpoint, get_timestep, - state, t_final, t=0.0, istep=0, logmgr=None, eos=None, dim=None): +def advance_state(rhs, timestepper, get_timestep, + state, t_final, t=0.0, istep=0, + pre_step_callback=None, + post_step_callback=None, + logmgr=None, eos=None, dim=None): """Advance state from some time (t) to some time (t_final). Parameters @@ -44,12 +47,7 @@ def _advance_state_stepper_func(rhs, timestepper, checkpoint, get_timestep, a call with signature ``rhs(t, state)``. timestepper Function that advances the state from t=time to t=(time+dt), and - returns the advanced state. Has a call with signature - ``timestepper(state, t, dt, rhs)``. - checkpoint - Function is user-defined and can be used to preform simulation status - reporting, viz, and restart i/o. A non-zero return code from this function - indicates that this function should stop gracefully. + returns the advanced state. get_timestep Function that should return dt for the next step. This interface allows user-defined adaptive timestepping. A negative return value indicated that @@ -63,6 +61,17 @@ def _advance_state_stepper_func(rhs, timestepper, checkpoint, get_timestep, Time at which to start istep: int Step number from which to start + pre_step_callback + An optional user-defined function to be called before the timestepper + is called for that particular step. A non-zero return code from this + function indicates that this function should stop gracefully. + Examples of such functions include visualization, io, or restart + checkpoints. + post_step_callback + An optional user-defined function to be called after the timestepper + is called for that particular step. A non-zero return code from this + function indicates that this function should stop gracefully. + Examples of such functions include applying modal filtering or limiters. Returns ------- @@ -84,11 +93,16 @@ def _advance_state_stepper_func(rhs, timestepper, checkpoint, get_timestep, if dt < 0: return istep, t, state - checkpoint(state=state, step=istep, t=t, dt=dt) + if pre_step_callback is not None: + pre_step_callback(state=state, step=istep, t=t, dt=dt) state = timestepper(state=state, t=t, dt=dt, rhs=rhs) t += dt + + if post_step_callback is not None: + post_step_callback(state=state, step=istep, t=t, dt=dt) + istep += 1 if logmgr: From 72ceaf609344dd00abd363dae904716bd2d916e4 Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Fri, 28 May 2021 13:46:02 -0500 Subject: [PATCH 110/385] Pass checkpoint functions as pre-step callbacks --- examples/autoignition-mpi.py | 6 +++--- examples/lump-mpi.py | 6 +++--- examples/mixture-mpi.py | 6 +++--- examples/pulse-mpi.py | 6 +++--- examples/scalar-lump-mpi.py | 6 +++--- examples/sod-mpi.py | 6 +++--- examples/vortex-mpi.py | 8 ++++---- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index caaef9821..a9f9983e5 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -242,9 +242,9 @@ def my_checkpoint(step, t, dt, state): try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=my_checkpoint, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final) except ExactSolutionMismatch as ex: error_state = True current_step = ex.step diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index e0272c57f..134214722 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -139,9 +139,9 @@ def my_checkpoint(step, t, dt, state): try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=my_checkpoint, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final) except ExactSolutionMismatch as ex: current_step = ex.step current_t = ex.t diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index b2d834c15..14b814b1b 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -162,9 +162,9 @@ def my_checkpoint(step, t, dt, state): try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=my_checkpoint, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final) except ExactSolutionMismatch as ex: error_state = 1 current_step = ex.step diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index f6ef9f649..33a53ecbf 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -162,9 +162,9 @@ def my_checkpoint(step, t, dt, state): try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=my_checkpoint, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final) except ExactSolutionMismatch as ex: current_step = ex.step current_t = ex.t diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 4ae61c2dd..90d387c29 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -149,9 +149,9 @@ def my_checkpoint(step, t, dt, state): try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=my_checkpoint, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final) except ExactSolutionMismatch as ex: current_step = ex.step current_t = ex.t diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 13d564b39..f8f61c186 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -138,9 +138,9 @@ def my_checkpoint(step, t, dt, state): try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=my_checkpoint, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final) except ExactSolutionMismatch as ex: current_step = ex.step current_t = ex.t diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 2149c4d5f..447f8b57c 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -186,10 +186,10 @@ def my_checkpoint(step, t, dt, state): try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=my_checkpoint, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final, logmgr=logmgr, - eos=eos, dim=dim) + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final, logmgr=logmgr, eos=eos, + dim=dim) except ExactSolutionMismatch as ex: current_step = ex.step current_t = ex.t From 28148262b99139c7ddfbbc49fe55e846e8996a54 Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Fri, 28 May 2021 16:08:16 -0500 Subject: [PATCH 111/385] Add healthcheck callback and exceptions --- doc/support/tools.rst | 2 +- mirgecom/exceptions.py | 78 ++++++++++++++++++++++++++++++++++++++++++ mirgecom/simutil.py | 53 +++++++++++++++++++--------- 3 files changed, 116 insertions(+), 17 deletions(-) create mode 100644 mirgecom/exceptions.py diff --git a/doc/support/tools.rst b/doc/support/tools.rst index a046e118a..f8e0b7c07 100644 --- a/doc/support/tools.rst +++ b/doc/support/tools.rst @@ -1,6 +1,6 @@ Random Pile'o'Tools =================== +.. automodule:: mirgecom.exceptions .. automodule:: mirgecom.simutil - .. automodule:: mirgecom.utils diff --git a/mirgecom/exceptions.py b/mirgecom/exceptions.py new file mode 100644 index 000000000..fb85af0e5 --- /dev/null +++ b/mirgecom/exceptions.py @@ -0,0 +1,78 @@ +"""Provide custom exceptions for use in callback routines. + +.. autoexception:: MirgecomException +.. autoexception:: ExactSolutionMismatch +.. autoexception:: SimulationHealthError +""" + +__copyright__ = """ +Copyright (C) 2021 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + + +class MirgecomException(Exception): + """Exception base class for mirgecom exceptions. + + .. attribute:: step + + A :class:`int` denoting the simulation step when the exception + was raised. + + .. attribute:: t + + A :class:`float` denoting the simulation time when the + exception was raised. + + .. attribute:: state + + The simulation state when the exception was raised. + + .. attribute:: message + + A :class:`str` describing the message for the exception. + """ + + def __init__(self, step, t, state, message): + """Record the simulation state on creation.""" + self.step = step + self.t = t + self.state = state + self.message = message + super().__init__(self.message) + + +class ExactSolutionMismatch(MirgecomException): + """Exception class for solution mismatch.""" + + def __init__(self, step, t, state): + super().__init__( + step, t, state, + message="Solution doesn't agree with analytic result." + ) + + +class SimulationHealthError(MirgecomException): + """Exception class for an unphysical simulation.""" + + def __init__(self, step, t, state, message): + super().__init__(step, t, state, message) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index bdc72d653..41cf51e8b 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -2,8 +2,8 @@ .. autofunction:: check_step .. autofunction:: inviscid_sim_timestep -.. autoexception:: ExactSolutionMismatch .. autofunction:: sim_checkpoint +.. autofunction:: sim_healthcheck .. autofunction:: generate_and_distribute_mesh """ @@ -37,6 +37,7 @@ from meshmode.dof_array import thaw from mirgecom.io import make_status_message from mirgecom.inviscid import get_inviscid_timestep # bad smell? +from mirgecom.exceptions import ExactSolutionMismatch, SimulationHealthError logger = logging.getLogger(__name__) @@ -74,21 +75,6 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, return mydt -class ExactSolutionMismatch(Exception): - """Exception class for solution mismatch. - - .. attribute:: step - .. attribute:: t - .. attribute:: state - """ - - def __init__(self, step, t, state): - """Record the simulation state on creation.""" - self.step = step - self.t = t - self.state = state - - def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, step=0, t=0, dt=0, cfl=1.0, nstatus=-1, nviz=-1, exittol=1e-16, constant_cfl=False, comm=None, viz_fields=None, overwrite=False, @@ -162,6 +148,41 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, raise ExactSolutionMismatch(step, t=t, state=q) +def sim_healthcheck(discr, eos, q, conserved_vars, step=0, t=0): + """Checks the health of the simulation by checking for unphysical values. + """ + import grudge.op as op + + # NOTE: Derived quantities are functions of the conserved variables. + # Therefore is it sufficient to check for unphysical values of + # temperature and pressure. + dependent_vars = eos.dependent_vars(conserved_vars) + + # Check for NaN + if (np.isnan(op.nodal_sum(discr, "vol", dependent_vars.pressure)) + or np.isnan(op.nodal_sum(discr, "vol", dependent_vars.temperature))): + raise SimulationHealthError( + step, t=t, state=q, + message="Detected a NaN." + ) + + # Check for non-positivity + if (op.nodal_min(discr, "vol", dependent_vars.pressure) < 0 + or op.nodal_min(discr, "vol", dependent_vars.temperature) < 0): + raise SimulationHealthError( + step, t=t, state=q, + message="Found non-positive values for pressure or temperature." + ) + + # Check for blow-up + if (op.norm(discr, dependent_vars.pressure, np.inf) == np.inf + or op.norm(discr, dependent_vars.temperature, np.inf) == np.inf): + raise SimulationHealthError( + step, t=t, state=q, + message="Infinity-norm of derived quantities is not finite." + ) + + def generate_and_distribute_mesh(comm, generate_mesh): """Generate a mesh and distribute it among all ranks in *comm*. From 25dd51c961f18e8aaec72ab69bb411dd726cf5e7 Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Fri, 28 May 2021 16:08:42 -0500 Subject: [PATCH 112/385] Use healthcheck callbacks in examples --- examples/autoignition-mpi.py | 15 +++++++++++---- examples/lump-mpi.py | 13 ++++++++++--- examples/mixture-mpi.py | 13 ++++++++++--- examples/pulse-mpi.py | 13 ++++++++++--- examples/scalar-lump-mpi.py | 14 +++++++++++--- examples/sod-mpi.py | 14 +++++++++++--- examples/vortex-mpi.py | 14 +++++++++++--- 7 files changed, 74 insertions(+), 22 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index a9f9983e5..43980ded1 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -40,10 +40,12 @@ from mirgecom.simutil import ( inviscid_sim_timestep, sim_checkpoint, + sim_healthcheck, check_step, - generate_and_distribute_mesh, - ExactSolutionMismatch + generate_and_distribute_mesh ) +from mirgecom.exceptions import MirgecomException +from mirgecom.fluid import split_conserved from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -52,7 +54,7 @@ from mirgecom.boundary import AdiabaticSlipBoundary from mirgecom.initializers import MixtureInitializer from mirgecom.eos import PyrometheusMixture -from mirgecom.euler import split_conserved + import cantera import pyrometheus as pyro @@ -239,13 +241,18 @@ def my_checkpoint(step, t, dt, state): constant_cfl=constant_cfl, comm=comm, viz_fields=viz_fields) + def my_simhealthcheck(state, step, t, dt): + cv = split_conserved(discr.dim, state) + sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_checkpoint, + post_step_callback=my_simhealthcheck, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final) - except ExactSolutionMismatch as ex: + except MirgecomException as ex: error_state = True current_step = ex.step current_t = ex.t diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 134214722..0ce1d85e7 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -40,9 +40,11 @@ from mirgecom.simutil import ( inviscid_sim_timestep, sim_checkpoint, - generate_and_distribute_mesh, - ExactSolutionMismatch + sim_healthcheck, + generate_and_distribute_mesh ) +from mirgecom.exceptions import MirgecomException +from mirgecom.fluid import split_conserved from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -136,13 +138,18 @@ def my_checkpoint(step, t, dt, state): t=t, dt=dt, nstatus=nstatus, nviz=nviz, exittol=exittol, constant_cfl=constant_cfl, comm=comm) + def my_simhealthcheck(state, step, t, dt): + cv = split_conserved(discr.dim, state) + sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_checkpoint, + post_step_callback=my_simhealthcheck, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final) - except ExactSolutionMismatch as ex: + except MirgecomException as ex: current_step = ex.step current_t = ex.t current_state = ex.state diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 14b814b1b..d61209f0d 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -40,9 +40,11 @@ from mirgecom.simutil import ( inviscid_sim_timestep, sim_checkpoint, - generate_and_distribute_mesh, - ExactSolutionMismatch + sim_healthcheck, + generate_and_distribute_mesh ) +from mirgecom.exceptions import MirgecomException +from mirgecom.fluid import split_conserved from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -159,13 +161,18 @@ def my_checkpoint(step, t, dt, state): t=t, dt=dt, nstatus=nstatus, nviz=nviz, exittol=exittol, constant_cfl=constant_cfl, comm=comm) + def my_simhealthcheck(state, step, t, dt): + cv = split_conserved(discr.dim, state) + sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_checkpoint, + post_step_callback=my_simhealthcheck, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final) - except ExactSolutionMismatch as ex: + except MirgecomException as ex: error_state = 1 current_step = ex.step current_t = ex.t diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 33a53ecbf..6620ef918 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -42,9 +42,10 @@ inviscid_sim_timestep, generate_and_distribute_mesh, sim_checkpoint, - ExactSolutionMismatch, + sim_healthcheck ) - +from mirgecom.exceptions import MirgecomException +from mirgecom.fluid import split_conserved from mirgecom.io import make_init_message from mirgecom.integrators import rk4_step @@ -159,13 +160,19 @@ def my_checkpoint(step, t, dt, state): t=t, dt=dt, nstatus=nstatus, nviz=nviz, exittol=exittol, constant_cfl=constant_cfl, comm=comm) + def my_simhealthcheck(state, step, t, dt): + cv = split_conserved(discr.dim, state) + sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_checkpoint, + post_step_callback=my_simhealthcheck, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final) - except ExactSolutionMismatch as ex: + + except MirgecomException as ex: current_step = ex.step current_t = ex.t current_state = ex.state diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 90d387c29..1f4c70b03 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -41,9 +41,11 @@ from mirgecom.simutil import ( inviscid_sim_timestep, sim_checkpoint, - generate_and_distribute_mesh, - ExactSolutionMismatch + sim_healthcheck, + generate_and_distribute_mesh ) +from mirgecom.exceptions import MirgecomException +from mirgecom.fluid import split_conserved from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -146,13 +148,19 @@ def my_checkpoint(step, t, dt, state): t=t, dt=dt, nstatus=nstatus, nviz=nviz, exittol=exittol, constant_cfl=constant_cfl, comm=comm) + def my_simhealthcheck(state, step, t, dt): + cv = split_conserved(discr.dim, state) + sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_checkpoint, + post_step_callback=my_simhealthcheck, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final) - except ExactSolutionMismatch as ex: + + except MirgecomException as ex: current_step = ex.step current_t = ex.t current_state = ex.state diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index f8f61c186..3b4547f20 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -39,9 +39,11 @@ from mirgecom.simutil import ( inviscid_sim_timestep, sim_checkpoint, - generate_and_distribute_mesh, - ExactSolutionMismatch, + sim_healthcheck, + generate_and_distribute_mesh ) +from mirgecom.exceptions import MirgecomException +from mirgecom.fluid import split_conserved from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -135,13 +137,19 @@ def my_checkpoint(step, t, dt, state): t=t, dt=dt, nstatus=nstatus, nviz=nviz, exittol=exittol, constant_cfl=constant_cfl, comm=comm) + def my_simhealthcheck(state, step, t, dt): + cv = split_conserved(discr.dim, state) + sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_checkpoint, + post_step_callback=my_simhealthcheck, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final) - except ExactSolutionMismatch as ex: + + except MirgecomException as ex: current_step = ex.step current_t = ex.t current_state = ex.state diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 447f8b57c..af3240d51 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -41,9 +41,11 @@ from mirgecom.simutil import ( inviscid_sim_timestep, sim_checkpoint, - generate_and_distribute_mesh, - ExactSolutionMismatch, + sim_healthcheck, + generate_and_distribute_mesh ) +from mirgecom.exceptions import MirgecomException +from mirgecom.fluid import split_conserved from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -183,14 +185,20 @@ def my_checkpoint(step, t, dt, state): exittol=exittol, constant_cfl=constant_cfl, comm=comm, vis_timer=vis_timer) + def my_simhealthcheck(state, step, t, dt): + cv = split_conserved(discr.dim, state) + sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_checkpoint, + post_step_callback=my_simhealthcheck, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, logmgr=logmgr, eos=eos, dim=dim) - except ExactSolutionMismatch as ex: + + except MirgecomException as ex: current_step = ex.step current_t = ex.t current_state = ex.state From 7ee673c092b912651b6f68eee5d4bac73f48b0bf Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Fri, 28 May 2021 16:15:01 -0500 Subject: [PATCH 113/385] Fix docstring for sim_healthcheck --- mirgecom/simutil.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 41cf51e8b..5d5bc4b0f 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -149,8 +149,7 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, def sim_healthcheck(discr, eos, q, conserved_vars, step=0, t=0): - """Checks the health of the simulation by checking for unphysical values. - """ + """Check the health of the simulation by checking for unphysical values.""" import grudge.op as op # NOTE: Derived quantities are functions of the conserved variables. From 9dba4ff1a5a63fd5170f0f25c3e0524a2ea7c292 Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Fri, 4 Jun 2021 11:45:22 -0500 Subject: [PATCH 114/385] Document callback signature and return state --- examples/autoignition-mpi.py | 4 ++- examples/lump-mpi.py | 4 ++- examples/mixture-mpi.py | 4 ++- examples/pulse-mpi.py | 4 ++- examples/scalar-lump-mpi.py | 4 ++- examples/sod-mpi.py | 4 ++- examples/vortex-mpi.py | 4 ++- mirgecom/simutil.py | 6 ++++- mirgecom/steppers.py | 46 ++++++++++++++++++++++++++--------- test/test_time_integrators.py | 8 +++--- 10 files changed, 64 insertions(+), 24 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 43980ded1..8b8fadaa1 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -243,7 +243,9 @@ def my_checkpoint(step, t, dt, state): def my_simhealthcheck(state, step, t, dt): cv = split_conserved(discr.dim, state) - sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + return sim_healthcheck(discr, eos, q=state, + conserved_vars=cv, + step=step, t=t) try: (current_step, current_t, current_state) = \ diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 0ce1d85e7..dab984d07 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -140,7 +140,9 @@ def my_checkpoint(step, t, dt, state): def my_simhealthcheck(state, step, t, dt): cv = split_conserved(discr.dim, state) - sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + return sim_healthcheck(discr, eos, q=state, + conserved_vars=cv, + step=step, t=t) try: (current_step, current_t, current_state) = \ diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index d61209f0d..80ddc93c3 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -163,7 +163,9 @@ def my_checkpoint(step, t, dt, state): def my_simhealthcheck(state, step, t, dt): cv = split_conserved(discr.dim, state) - sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + return sim_healthcheck(discr, eos, q=state, + conserved_vars=cv, + step=step, t=t) try: (current_step, current_t, current_state) = \ diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 6620ef918..6b42aa16e 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -162,7 +162,9 @@ def my_checkpoint(step, t, dt, state): def my_simhealthcheck(state, step, t, dt): cv = split_conserved(discr.dim, state) - sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + return sim_healthcheck(discr, eos, q=state, + conserved_vars=cv, + step=step, t=t) try: (current_step, current_t, current_state) = \ diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 1f4c70b03..faae596b5 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -150,7 +150,9 @@ def my_checkpoint(step, t, dt, state): def my_simhealthcheck(state, step, t, dt): cv = split_conserved(discr.dim, state) - sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + return sim_healthcheck(discr, eos, q=state, + conserved_vars=cv, + step=step, t=t) try: (current_step, current_t, current_state) = \ diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 3b4547f20..5df71bb97 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -139,7 +139,9 @@ def my_checkpoint(step, t, dt, state): def my_simhealthcheck(state, step, t, dt): cv = split_conserved(discr.dim, state) - sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + return sim_healthcheck(discr, eos, q=state, + conserved_vars=cv, + step=step, t=t) try: (current_step, current_t, current_state) = \ diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index af3240d51..9ee849062 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -187,7 +187,9 @@ def my_checkpoint(step, t, dt, state): def my_simhealthcheck(state, step, t, dt): cv = split_conserved(discr.dim, state) - sim_healthcheck(discr, eos, q=state, conserved_vars=cv, step=step, t=t) + return sim_healthcheck(discr, eos, q=state, + conserved_vars=cv, + step=step, t=t) try: (current_step, current_t, current_state) = \ diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 5d5bc4b0f..99e57c6b9 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -83,7 +83,7 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, do_viz = check_step(step=step, interval=nviz) do_status = check_step(step=step, interval=nstatus) if do_viz is False and do_status is False: - return 0 + return q from mirgecom.fluid import split_conserved cv = split_conserved(discr.dim, q) @@ -147,6 +147,8 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, if maxerr > exittol: raise ExactSolutionMismatch(step, t=t, state=q) + return q + def sim_healthcheck(discr, eos, q, conserved_vars, step=0, t=0): """Check the health of the simulation by checking for unphysical values.""" @@ -181,6 +183,8 @@ def sim_healthcheck(discr, eos, q, conserved_vars, step=0, t=0): message="Infinity-norm of derived quantities is not finite." ) + return q + def generate_and_distribute_mesh(comm, generate_mesh): """Generate a mesh and distribute it among all ranks in *comm*. diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index e8074b9db..0ca3af0b6 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -62,16 +62,13 @@ def advance_state(rhs, timestepper, get_timestep, istep: int Step number from which to start pre_step_callback - An optional user-defined function to be called before the timestepper - is called for that particular step. A non-zero return code from this - function indicates that this function should stop gracefully. - Examples of such functions include visualization, io, or restart - checkpoints. + An optional user-defined function, with signature: + ``state = pre_step_callback(state, step, t, dt)``, + to be called before the timestepper is called for that particular step. post_step_callback - An optional user-defined function to be called after the timestepper - is called for that particular step. A non-zero return code from this - function indicates that this function should stop gracefully. - Examples of such functions include applying modal filtering or limiters. + An optional user-defined function, with signature: + ``state = post_step_callback(state, step, t, dt)``, + to be called after the timestepper is called for that particular step. Returns ------- @@ -94,14 +91,14 @@ def advance_state(rhs, timestepper, get_timestep, return istep, t, state if pre_step_callback is not None: - pre_step_callback(state=state, step=istep, t=t, dt=dt) + state = pre_step_callback(state=state, step=istep, t=t, dt=dt) state = timestepper(state=state, t=t, dt=dt, rhs=rhs) t += dt if post_step_callback is not None: - post_step_callback(state=state, step=istep, t=t, dt=dt) + state = post_step_callback(state=state, step=istep, t=t, dt=dt) istep += 1 @@ -145,6 +142,14 @@ def _advance_state_leap(rhs, timestepper, checkpoint, get_timestep, Time at which to start istep: int Step number from which to start + pre_step_callback + An optional user-defined function, with signature: + ``state = pre_step_callback(state, step, t, dt)``, + to be called before the timestepper is called for that particular step. + post_step_callback + An optional user-defined function, with signature: + ``state = post_step_callback(state, step, t, dt)``, + to be called after the timestepper is called for that particular step. Returns ------- @@ -170,13 +175,22 @@ def _advance_state_leap(rhs, timestepper, checkpoint, get_timestep, if dt < 0: return istep, t, state - checkpoint(state=state, step=istep, t=t, dt=dt) + if pre_step_callback is not None: + state = pre_step_callback(state=state, + step=istep, + t=t, dt=dt) # Leap interface here is *a bit* different. for event in stepper_cls.run(t_end=t+dt): if isinstance(event, stepper_cls.StateComputed): state = event.state_component t += dt + + if post_step_callback is not None: + state = post_step_callback(state=state, + step=istep, + t=t, dt=dt) + istep += 1 if logmgr: set_dt(logmgr, dt) @@ -263,6 +277,14 @@ def advance_state(rhs, timestepper, checkpoint, get_timestep, state, t_final, Time at which to start istep: int Step number from which to start + pre_step_callback + An optional user-defined function, with signature: + ``state = pre_step_callback(state, step, t, dt)``, + to be called before the timestepper is called for that particular step. + post_step_callback + An optional user-defined function, with signature: + ``state = post_step_callback(state, step, t, dt)``, + to be called after the timestepper is called for that particular step. Returns ------- diff --git a/test/test_time_integrators.py b/test/test_time_integrators.py index fe69d7514..cc55ceeca 100644 --- a/test/test_time_integrators.py +++ b/test/test_time_integrators.py @@ -128,13 +128,13 @@ def get_timestep(state): return dt def my_checkpoint(state, step, t, dt): - return 0 + return state (step, t, state) = \ advance_state(rhs=rhs, timestepper=method, - checkpoint=my_checkpoint, - get_timestep=get_timestep, state=state, - t=t, t_final=t_final, component_id="y") + get_timestep=get_timestep, state=state, + t=t, t_final=t_final, component_id="y", + post_step_callback=my_checkpoint) error = np.abs(state - exact_soln(t)) / exact_soln(t) integrator_eoc.add_data_point(dt, error) From f1b3e4f3289a5d9f6837f9772e25d9649cf62b0a Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Fri, 4 Jun 2021 12:37:20 -0500 Subject: [PATCH 115/385] Test the basic healthcheck callback --- test/test_callbacks.py | 100 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 test/test_callbacks.py diff --git a/test/test_callbacks.py b/test/test_callbacks.py new file mode 100644 index 000000000..fb05c445e --- /dev/null +++ b/test/test_callbacks.py @@ -0,0 +1,100 @@ +"""Test built-in callback routines.""" + +__copyright__ = """ +Copyright (C) 2021 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import numpy as np +import pytest + +from arraycontext import ( # noqa + thaw, + pytest_generate_tests_for_pyopencl_array_context + as pytest_generate_tests +) + +from mirgecom.fluid import join_conserved, split_conserved +from mirgecom.eos import IdealSingleGas + +from grudge.eager import EagerDGDiscretization + + +def test_basic_healthcheck(actx_factory): + from mirgecom.simutil import sim_healthcheck + + actx = actx_factory() + nel_1d = 4 + dim = 2 + + from meshmode.mesh.generation import generate_regular_rect_mesh + + mesh = generate_regular_rect_mesh( + a=(1.0,) * dim, b=(2.0,) * dim, nelements_per_axis=(nel_1d,) * dim + ) + + order = 3 + discr = EagerDGDiscretization(actx, mesh, order=order) + nodes = thaw(discr.nodes(), actx) + zeros = discr.zeros(actx) + ones = zeros + 1.0 + + # Let's make a very bad state (negative mass) + mass = -1*ones + energy = zeros + 2.5 + velocity = 2 * nodes + mom = mass * velocity + + eos = IdealSingleGas() + q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) + cv = split_conserved(dim, q) + + from mirgecom.exceptions import SimulationHealthError + + with pytest.raises(SimulationHealthError): + sim_healthcheck(discr, eos, q, cv) + + # Let's make another very bad state (nans) + mass = 1*ones + energy = zeros + 2.5 + velocity = np.nan * nodes + mom = mass * velocity + + eos = IdealSingleGas() + q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) + cv = split_conserved(dim, q) + + with pytest.raises(SimulationHealthError): + sim_healthcheck(discr, eos, q, cv) + + # Let's make one last very bad state (inf) + mass = 1*ones + energy = np.inf * ones + velocity = 2 * nodes + mom = mass * velocity + + eos = IdealSingleGas() + q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) + cv = split_conserved(dim, q) + + with pytest.raises(SimulationHealthError): + sim_healthcheck(discr, eos, q, cv) From faba167c3930c6798243c75176d5fb6cd29a26de Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Fri, 4 Jun 2021 12:43:30 -0500 Subject: [PATCH 116/385] Remove EOS copy-pasta --- test/test_callbacks.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test_callbacks.py b/test/test_callbacks.py index fb05c445e..ebdf12aa6 100644 --- a/test/test_callbacks.py +++ b/test/test_callbacks.py @@ -79,7 +79,6 @@ def test_basic_healthcheck(actx_factory): velocity = np.nan * nodes mom = mass * velocity - eos = IdealSingleGas() q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) cv = split_conserved(dim, q) @@ -92,7 +91,6 @@ def test_basic_healthcheck(actx_factory): velocity = 2 * nodes mom = mass * velocity - eos = IdealSingleGas() q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) cv = split_conserved(dim, q) From 0b6df95ef9df205115d65947fd40bee6da505ee4 Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Sat, 5 Jun 2021 10:41:20 -0500 Subject: [PATCH 117/385] Minor refactoring and allow callbacks to terminate the simulation --- examples/autoignition-mpi.py | 71 +++++++------- examples/lump-mpi.py | 63 +++++++------ examples/mixture-mpi.py | 70 +++++++------- examples/pulse-mpi.py | 66 +++++++------ examples/scalar-lump-mpi.py | 64 +++++++------ examples/sod-mpi.py | 64 +++++++------ examples/vortex-mpi.py | 68 +++++++------ mirgecom/exceptions.py | 40 +------- mirgecom/simutil.py | 178 +++++++++++++++++++++++------------ test/test_callbacks.py | 15 ++- 10 files changed, 377 insertions(+), 322 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 8b8fadaa1..5881651ea 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -39,12 +39,13 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, + sim_visualization, sim_checkpoint, - sim_healthcheck, + sim_cfd_healthcheck, check_step, generate_and_distribute_mesh ) -from mirgecom.exceptions import MirgecomException +from mirgecom.exceptions import StepperCrashError from mirgecom.fluid import split_conserved from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -84,6 +85,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 1 nviz = 5 + ncheck = 1 rank = 0 checkpoint_t = current_t current_step = 0 @@ -94,7 +96,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): timestepper = rk4_step box_ll = -0.005 box_ur = 0.005 - error_state = False debug = False from mpi4py import MPI @@ -232,33 +233,37 @@ def my_rhs(t, state): + eos.get_species_source_terms(cv)) def my_checkpoint(step, t, dt, state): - cv = split_conserved(dim, state) - reaction_rates = eos.get_production_rates(cv) - viz_fields = [("reaction_rates", reaction_rates)] - return sim_checkpoint(discr, visualizer, eos, q=state, - vizname=casename, step=step, - t=t, dt=dt, nstatus=nstatus, nviz=nviz, - constant_cfl=constant_cfl, comm=comm, - viz_fields=viz_fields) - - def my_simhealthcheck(state, step, t, dt): - cv = split_conserved(discr.dim, state) - return sim_healthcheck(discr, eos, q=state, - conserved_vars=cv, - step=step, t=t) - - try: - (current_step, current_t, current_state) = \ - advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, - post_step_callback=my_simhealthcheck, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) - except MirgecomException as ex: - error_state = True - current_step = ex.step - current_t = ex.t - current_state = ex.state + try: + # Check the health of the simulation + sim_cfd_healthcheck(discr, eos, q=state, + ncheck=ncheck, step=step, t=t) + # Perform checkpointing + sim_checkpoint(discr, eos, q=state, + exact_soln=initializer, + step=step, t=t, dt=dt, + nstatus=nstatus, + constant_cfl=constant_cfl) + # Visualize + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=nviz) + except StepperCrashError as err: + # Log crash error message + if rank == 0: + logger.info(str(err)) + logger.info("Visualizing crashed state ...") + # Write out crashed field + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=1) + raise err + return state + + current_step, current_t, current_state = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final, eos=eos, dim=dim) if not check_step(current_step, nviz): # If final step not an output step if rank == 0: @@ -267,12 +272,6 @@ def my_simhealthcheck(state, step, t, dt): dt=(current_t - checkpoint_t), state=current_state) - if current_t - t_final < 0: - error_state = True - - if error_state: - raise ValueError("Simulation did not complete successfully.") - if __name__ == "__main__": logging.basicConfig(level=logging.INFO) diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index dab984d07..44130fa1c 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -39,12 +39,12 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, + sim_visualization, sim_checkpoint, - sim_healthcheck, + sim_cfd_healthcheck, generate_and_distribute_mesh ) -from mirgecom.exceptions import MirgecomException -from mirgecom.fluid import split_conserved +from mirgecom.exceptions import StepperCrashError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -84,6 +84,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 1 nviz = 1 + ncheck = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -133,28 +134,37 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - return sim_checkpoint(discr, visualizer, eos, q=state, - exact_soln=initializer, vizname=casename, step=step, - t=t, dt=dt, nstatus=nstatus, nviz=nviz, - exittol=exittol, constant_cfl=constant_cfl, comm=comm) - - def my_simhealthcheck(state, step, t, dt): - cv = split_conserved(discr.dim, state) - return sim_healthcheck(discr, eos, q=state, - conserved_vars=cv, - step=step, t=t) - - try: - (current_step, current_t, current_state) = \ - advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, - post_step_callback=my_simhealthcheck, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) - except MirgecomException as ex: - current_step = ex.step - current_t = ex.t - current_state = ex.state + try: + # Check the health of the simulation + sim_cfd_healthcheck(discr, eos, q=state, + ncheck=ncheck, step=step, t=t) + # Perform checkpointing + sim_checkpoint(discr, eos, q=state, + exact_soln=initializer, + step=step, t=t, dt=dt, + nstatus=nstatus, exittol=exittol, + constant_cfl=constant_cfl) + # Visualize + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=nviz) + except StepperCrashError as err: + # Log crash error message + if rank == 0: + logger.info(str(err)) + logger.info("Visualizing crashed state ...") + # Write out crashed field + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=1) + raise err + return state + + current_step, current_t, current_state = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final, eos=eos, dim=dim) # if current_t != checkpoint_t: if rank == 0: @@ -163,9 +173,6 @@ def my_simhealthcheck(state, step, t, dt): dt=(current_t - checkpoint_t), state=current_state) - if current_t - t_final < 0: - raise ValueError("Simulation exited abnormally") - if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 80ddc93c3..3b5bc921e 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -39,12 +39,12 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, + sim_visualization, sim_checkpoint, - sim_healthcheck, + sim_cfd_healthcheck, generate_and_distribute_mesh ) -from mirgecom.exceptions import MirgecomException -from mirgecom.fluid import split_conserved +from mirgecom.exceptions import StepperCrashError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -81,6 +81,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 1 nviz = 1 + ncheck = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -91,7 +92,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): timestepper = rk4_step box_ll = -5.0 box_ur = 5.0 - error_state = 0 from mpi4py import MPI comm = MPI.COMM_WORLD @@ -154,31 +154,37 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - global checkpoint_t - checkpoint_t = t - return sim_checkpoint(discr, visualizer, eos, q=state, - exact_soln=initializer, vizname=casename, step=step, - t=t, dt=dt, nstatus=nstatus, nviz=nviz, - exittol=exittol, constant_cfl=constant_cfl, comm=comm) - - def my_simhealthcheck(state, step, t, dt): - cv = split_conserved(discr.dim, state) - return sim_healthcheck(discr, eos, q=state, - conserved_vars=cv, - step=step, t=t) - - try: - (current_step, current_t, current_state) = \ - advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, - post_step_callback=my_simhealthcheck, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) - except MirgecomException as ex: - error_state = 1 - current_step = ex.step - current_t = ex.t - current_state = ex.state + try: + # Check the health of the simulation + sim_cfd_healthcheck(discr, eos, q=state, + ncheck=ncheck, step=step, t=t) + # Perform checkpointing + sim_checkpoint(discr, eos, q=state, + exact_soln=initializer, + step=step, t=t, dt=dt, + nstatus=nstatus, exittol=exittol, + constant_cfl=constant_cfl) + # Visualize + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=nviz) + except StepperCrashError as err: + # Log crash error message + if rank == 0: + logger.info(str(err)) + logger.info("Visualizing crashed state ...") + # Write out crashed field + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=1) + raise err + return state + + current_step, current_t, current_state = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final, eos=eos, dim=dim) if current_t != checkpoint_t: # This check because !overwrite if rank == 0: @@ -187,12 +193,6 @@ def my_simhealthcheck(state, step, t, dt): dt=(current_t - checkpoint_t), state=current_state) - if current_t - t_final < 0: - error_state = 1 - - if error_state: - raise ValueError("Simulation did not complete successfully.") - if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 6b42aa16e..74f1e0366 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -40,12 +40,12 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, - generate_and_distribute_mesh, + sim_visualization, sim_checkpoint, - sim_healthcheck + sim_cfd_healthcheck, + generate_and_distribute_mesh ) -from mirgecom.exceptions import MirgecomException -from mirgecom.fluid import split_conserved +from mirgecom.exceptions import StepperCrashError from mirgecom.io import make_init_message from mirgecom.integrators import rk4_step @@ -92,6 +92,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 10 nviz = 10 + ncheck = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -155,29 +156,37 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - return sim_checkpoint(discr, visualizer, eos, q=state, - vizname=casename, step=step, - t=t, dt=dt, nstatus=nstatus, nviz=nviz, - exittol=exittol, constant_cfl=constant_cfl, comm=comm) - - def my_simhealthcheck(state, step, t, dt): - cv = split_conserved(discr.dim, state) - return sim_healthcheck(discr, eos, q=state, - conserved_vars=cv, - step=step, t=t) - - try: - (current_step, current_t, current_state) = \ - advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, - post_step_callback=my_simhealthcheck, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) - - except MirgecomException as ex: - current_step = ex.step - current_t = ex.t - current_state = ex.state + try: + # Check the health of the simulation + sim_cfd_healthcheck(discr, eos, q=state, + ncheck=ncheck, step=step, t=t) + # Perform checkpointing + sim_checkpoint(discr, eos, q=state, + exact_soln=initializer, + step=step, t=t, dt=dt, + nstatus=nstatus, exittol=exittol, + constant_cfl=constant_cfl) + # Visualize + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=nviz) + except StepperCrashError as err: + # Log crash error message + if rank == 0: + logger.info(str(err)) + logger.info("Visualizing crashed state ...") + # Write out crashed field + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=1) + raise err + return state + + current_step, current_t, current_state = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final, eos=eos, dim=dim) # if current_t != checkpoint_t: if rank == 0: @@ -186,9 +195,6 @@ def my_simhealthcheck(state, step, t, dt): dt=(current_t - checkpoint_t), state=current_state) - if current_t - t_final < 0: - raise ValueError("Simulation exited abnormally") - if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index faae596b5..a8dc1696b 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -40,12 +40,12 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, + sim_visualization, sim_checkpoint, - sim_healthcheck, + sim_cfd_healthcheck, generate_and_distribute_mesh ) -from mirgecom.exceptions import MirgecomException -from mirgecom.fluid import split_conserved +from mirgecom.exceptions import StepperCrashError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -78,6 +78,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 1 nviz = 1 + ncheck = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -143,29 +144,37 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - return sim_checkpoint(discr, visualizer, eos, q=state, - exact_soln=initializer, vizname=casename, step=step, - t=t, dt=dt, nstatus=nstatus, nviz=nviz, - exittol=exittol, constant_cfl=constant_cfl, comm=comm) - - def my_simhealthcheck(state, step, t, dt): - cv = split_conserved(discr.dim, state) - return sim_healthcheck(discr, eos, q=state, - conserved_vars=cv, - step=step, t=t) - - try: - (current_step, current_t, current_state) = \ - advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, - post_step_callback=my_simhealthcheck, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) - - except MirgecomException as ex: - current_step = ex.step - current_t = ex.t - current_state = ex.state + try: + # Check the health of the simulation + sim_cfd_healthcheck(discr, eos, q=state, + ncheck=ncheck, step=step, t=t) + # Perform checkpointing + sim_checkpoint(discr, eos, q=state, + exact_soln=initializer, + step=step, t=t, dt=dt, + nstatus=nstatus, exittol=exittol, + constant_cfl=constant_cfl) + # Visualize + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=nviz) + except StepperCrashError as err: + # Log crash error message + if rank == 0: + logger.info(str(err)) + logger.info("Visualizing crashed state ...") + # Write out crashed field + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=1) + raise err + return state + + current_step, current_t, current_state = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final, eos=eos, dim=dim) # if current_t != checkpoint_t: if rank == 0: @@ -174,9 +183,6 @@ def my_simhealthcheck(state, step, t, dt): dt=(current_t - checkpoint_t), state=current_state) - if current_t - t_final < 0: - raise ValueError("Simulation exited abnormally") - if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 5df71bb97..84fb47a4d 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -38,12 +38,12 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, + sim_visualization, sim_checkpoint, - sim_healthcheck, + sim_cfd_healthcheck, generate_and_distribute_mesh ) -from mirgecom.exceptions import MirgecomException -from mirgecom.fluid import split_conserved +from mirgecom.exceptions import StepperCrashError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -81,6 +81,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 10 nviz = 10 + ncheck = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -132,29 +133,37 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - return sim_checkpoint(discr, visualizer, eos, q=state, - exact_soln=initializer, vizname=casename, step=step, - t=t, dt=dt, nstatus=nstatus, nviz=nviz, - exittol=exittol, constant_cfl=constant_cfl, comm=comm) - - def my_simhealthcheck(state, step, t, dt): - cv = split_conserved(discr.dim, state) - return sim_healthcheck(discr, eos, q=state, - conserved_vars=cv, - step=step, t=t) - - try: - (current_step, current_t, current_state) = \ - advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, - post_step_callback=my_simhealthcheck, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) - - except MirgecomException as ex: - current_step = ex.step - current_t = ex.t - current_state = ex.state + try: + # Check the health of the simulation + sim_cfd_healthcheck(discr, eos, q=state, + ncheck=ncheck, step=step, t=t) + # Perform checkpointing + sim_checkpoint(discr, eos, q=state, + exact_soln=initializer, + step=step, t=t, dt=dt, + nstatus=nstatus, exittol=exittol, + constant_cfl=constant_cfl) + # Visualize + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=nviz) + except StepperCrashError as err: + # Log crash error message + if rank == 0: + logger.info(str(err)) + logger.info("Visualizing crashed state ...") + # Write out crashed field + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=1) + raise err + return state + + current_step, current_t, current_state = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final, eos=eos, dim=dim) # if current_t != checkpoint_t: if rank == 0: @@ -163,9 +172,6 @@ def my_simhealthcheck(state, step, t, dt): dt=(current_t - checkpoint_t), state=current_state) - if current_t - t_final < 0: - raise ValueError("Simulation exited abnormally") - if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 9ee849062..5014582c2 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -40,12 +40,12 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, + sim_visualization, sim_checkpoint, - sim_healthcheck, + sim_cfd_healthcheck, generate_and_distribute_mesh ) -from mirgecom.exceptions import MirgecomException -from mirgecom.fluid import split_conserved +from mirgecom.exceptions import StepperCrashError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -106,6 +106,7 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal constant_cfl = False nstatus = 10 nviz = 10 + ncheck = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -179,42 +180,47 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - return sim_checkpoint(discr, visualizer, eos, q=state, - exact_soln=initializer, vizname=casename, step=step, - t=t, dt=dt, nstatus=nstatus, nviz=nviz, - exittol=exittol, constant_cfl=constant_cfl, comm=comm, + try: + # Check the health of the simulation + sim_cfd_healthcheck(discr, eos, q=state, + ncheck=ncheck, step=step, t=t) + # Perform checkpointing + sim_checkpoint(discr, eos, q=state, + exact_soln=initializer, + step=step, t=t, dt=dt, + nstatus=nstatus, exittol=exittol, + constant_cfl=constant_cfl) + # Visualize + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=nviz, + vis_timer=vis_timer) + except StepperCrashError as err: + # Log crash error message + if rank == 0: + logger.info(str(err)) + logger.info("Visualizing crashed state ...") + # Write out crashed field + sim_visualization(discr, state, eos, + visualizer, vizname=casename, + step=step, t=t, nviz=1, vis_timer=vis_timer) + raise err + return state + + current_step, current_t, current_state = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + pre_step_callback=my_checkpoint, + get_timestep=get_timestep, state=current_state, + t=current_t, t_final=t_final, + logmgr=logmgr, eos=eos, dim=dim) - def my_simhealthcheck(state, step, t, dt): - cv = split_conserved(discr.dim, state) - return sim_healthcheck(discr, eos, q=state, - conserved_vars=cv, - step=step, t=t) - - try: - (current_step, current_t, current_state) = \ - advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, - post_step_callback=my_simhealthcheck, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final, logmgr=logmgr, eos=eos, - dim=dim) - - except MirgecomException as ex: - current_step = ex.step - current_t = ex.t - current_state = ex.state - - # if current_t != checkpoint_t: if rank == 0: logger.info("Checkpointing final state ...") my_checkpoint(current_step, t=current_t, dt=(current_t - checkpoint_t), state=current_state) - if current_t - t_final < 0: - raise ValueError("Simulation exited abnormally") - if logmgr: logmgr.close() elif use_profiling: diff --git a/mirgecom/exceptions.py b/mirgecom/exceptions.py index fb85af0e5..e53560b00 100644 --- a/mirgecom/exceptions.py +++ b/mirgecom/exceptions.py @@ -1,7 +1,6 @@ """Provide custom exceptions for use in callback routines. .. autoexception:: MirgecomException -.. autoexception:: ExactSolutionMismatch .. autoexception:: SimulationHealthError """ @@ -30,49 +29,18 @@ """ -class MirgecomException(Exception): - """Exception base class for mirgecom exceptions. - - .. attribute:: step - - A :class:`int` denoting the simulation step when the exception - was raised. - - .. attribute:: t - - A :class:`float` denoting the simulation time when the - exception was raised. - - .. attribute:: state - - The simulation state when the exception was raised. +class StepperCrashError(Exception): + """Exception base class for simulation exceptions. .. attribute:: message A :class:`str` describing the message for the exception. """ - def __init__(self, step, t, state, message): - """Record the simulation state on creation.""" - self.step = step - self.t = t - self.state = state + def __init__(self, message): self.message = message super().__init__(self.message) -class ExactSolutionMismatch(MirgecomException): - """Exception class for solution mismatch.""" - - def __init__(self, step, t, state): - super().__init__( - step, t, state, - message="Solution doesn't agree with analytic result." - ) - - -class SimulationHealthError(MirgecomException): +class SimulationHealthError(StepperCrashError): """Exception class for an unphysical simulation.""" - - def __init__(self, step, t, state, message): - super().__init__(step, t, state, message) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 99e57c6b9..7b36b7ed7 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -2,8 +2,9 @@ .. autofunction:: check_step .. autofunction:: inviscid_sim_timestep +.. autofunction:: sim_visualization .. autofunction:: sim_checkpoint -.. autofunction:: sim_healthcheck +.. autofunction:: sim_cfd_healthcheck .. autofunction:: generate_and_distribute_mesh """ @@ -37,7 +38,7 @@ from meshmode.dof_array import thaw from mirgecom.io import make_status_message from mirgecom.inviscid import get_inviscid_timestep # bad smell? -from mirgecom.exceptions import ExactSolutionMismatch, SimulationHealthError +from mirgecom.exceptions import SimulationHealthError logger = logging.getLogger(__name__) @@ -75,24 +76,72 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, return mydt -def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, - step=0, t=0, dt=0, cfl=1.0, nstatus=-1, nviz=-1, exittol=1e-16, - constant_cfl=False, comm=None, viz_fields=None, overwrite=False, - vis_timer=None): - """Check simulation health, status, viz dumps, and restart.""" - do_viz = check_step(step=step, interval=nviz) - do_status = check_step(step=step, interval=nstatus) - if do_viz is False and do_status is False: - return q - +def sim_visualization(discr, state, eos, visualizer, vizname, + step=0, t=0, nviz=-1, + exact_soln=None, viz_fields=None, + overwrite=False, vis_timer=None): + """Visualize the simulation fields.""" + from contextlib import nullcontext from mirgecom.fluid import split_conserved - cv = split_conserved(discr.dim, q) + from mirgecom.io import make_rank_fname, make_par_fname + + if not check_step(step=step, interval=nviz): + return + + cv = split_conserved(discr.dim, state) dependent_vars = eos.dependent_vars(cv) + io_fields = [ + ("cv", cv), + ("dv", dependent_vars) + ] + if exact_soln is not None: + actx = cv.mass.array_context + nodes = thaw(actx, discr.nodes()) + expected_state = exact_soln(x_vec=nodes, t=t, eos=eos) + exact_list = [ + ("exact_soln", expected_state), + ] + io_fields.extend(exact_list) + + if viz_fields is not None: + io_fields.extend(viz_fields) + + comm = discr.mpi_communicator rank = 0 if comm is not None: rank = comm.Get_rank() + rank_fn = make_rank_fname(basename=vizname, rank=rank, step=step, t=t) + + if vis_timer: + ctm = vis_timer.start_sub_timer() + else: + ctm = nullcontext() + + with ctm: + visualizer.write_parallel_vtk_file( + comm, rank_fn, io_fields, + overwrite=overwrite, + par_manifest_filename=make_par_fname( + basename=vizname, step=step, t=t + ) + ) + + +def sim_checkpoint(discr, eos, q, exact_soln=None, + step=0, t=0, dt=0, cfl=1.0, nstatus=-1, exittol=1e-16, + constant_cfl=False): + """Check simulation status, and restart.""" + from mirgecom.fluid import split_conserved + + do_status = check_step(step=step, interval=nstatus) + if do_status is False: + return + + cv = split_conserved(discr.dim, q) + dependent_vars = eos.dependent_vars(cv) + maxerr = 0.0 if exact_soln is not None: actx = cv.mass.array_context @@ -102,33 +151,10 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, err_norms = [discr.norm(v, np.inf) for v in exp_resid] maxerr = max(err_norms) - if do_viz: - io_fields = [ - ("cv", cv), - ("dv", dependent_vars) - ] - if exact_soln is not None: - exact_list = [ - ("exact_soln", expected_state), - ] - io_fields.extend(exact_list) - if viz_fields is not None: - io_fields.extend(viz_fields) - - from mirgecom.io import make_rank_fname, make_par_fname - rank_fn = make_rank_fname(basename=vizname, rank=rank, step=step, t=t) - - from contextlib import nullcontext - - if vis_timer: - ctm = vis_timer.start_sub_timer() - else: - ctm = nullcontext() - - with ctm: - visualizer.write_parallel_vtk_file(comm, rank_fn, io_fields, - overwrite=overwrite, par_manifest_filename=make_par_fname( - basename=vizname, step=step, t=t)) + comm = discr.mpi_communicator + rank = 0 + if comm is not None: + rank = comm.Get_rank() if do_status is True: # if constant_cfl is False: @@ -145,46 +171,80 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, logger.info(statusmesg) if maxerr > exittol: - raise ExactSolutionMismatch(step, t=t, state=q) + raise SimulationHealthError( + message=( + "Simulation exited abnormally; " + "solution doesn't agree with analytic result." + ) + ) + - return q +def sim_cfd_healthcheck(discr, eos, q, ncheck=-1, step=0, t=0): + """Check the global health of the fluids state *q. + Determine the health of a state by inspecting for unphyiscal + values of pressure and temperature. -def sim_healthcheck(discr, eos, q, conserved_vars, step=0, t=0): - """Check the health of the simulation by checking for unphysical values.""" + Parameters + ---------- + eos: mirgecom.eos.GasEOS + Implementing the pressure and temperature functions for + returning pressure and temperature as a function of the state q. + q + State array which expects at least the conserved quantities + (mass, energy, momentum) for the fluid at each point. For multi-component + fluids, the conserved quantities should include + (mass, energy, momentum, species_mass), where *species_mass* is a vector + of species masses. + ncheck: int + An integer denoting the frequency interval. + + Returns + ------- + None + """ + from mirgecom.fluid import split_conserved import grudge.op as op + if not check_step(step=step, interval=ncheck): + return + + cv = split_conserved(discr.dim, q) + # NOTE: Derived quantities are functions of the conserved variables. # Therefore is it sufficient to check for unphysical values of # temperature and pressure. - dependent_vars = eos.dependent_vars(conserved_vars) + dependent_vars = eos.dependent_vars(cv) # Check for NaN - if (np.isnan(op.nodal_sum(discr, "vol", dependent_vars.pressure)) - or np.isnan(op.nodal_sum(discr, "vol", dependent_vars.temperature))): + if (np.isnan(op.nodal_sum_loc(discr, "vol", dependent_vars.pressure)) + or np.isnan(op.nodal_sum_loc(discr, "vol", dependent_vars.temperature))): raise SimulationHealthError( - step, t=t, state=q, - message="Detected a NaN." + message=( + "Simulation exited abnormally; detected a NaN." + ) ) # Check for non-positivity - if (op.nodal_min(discr, "vol", dependent_vars.pressure) < 0 - or op.nodal_min(discr, "vol", dependent_vars.temperature) < 0): + if (op.nodal_min_loc(discr, "vol", dependent_vars.pressure) < 0 + or op.nodal_min_loc(discr, "vol", dependent_vars.temperature) < 0): raise SimulationHealthError( - step, t=t, state=q, - message="Found non-positive values for pressure or temperature." + message=( + "Simulation exited abnormally; " + "found non-positive values for pressure or temperature." + ) ) # Check for blow-up - if (op.norm(discr, dependent_vars.pressure, np.inf) == np.inf - or op.norm(discr, dependent_vars.temperature, np.inf) == np.inf): + if (op.nodal_sum_loc(discr, "vol", dependent_vars.pressure) == np.inf + or op.nodal_sum_loc(discr, "vol", dependent_vars.temperature) == np.inf): raise SimulationHealthError( - step, t=t, state=q, - message="Infinity-norm of derived quantities is not finite." + message=( + "Simulation exited abnormally; " + "derived quantities are not finite." + ) ) - return q - def generate_and_distribute_mesh(comm, generate_mesh): """Generate a mesh and distribute it among all ranks in *comm*. diff --git a/test/test_callbacks.py b/test/test_callbacks.py index ebdf12aa6..72d9ff82a 100644 --- a/test/test_callbacks.py +++ b/test/test_callbacks.py @@ -33,14 +33,14 @@ as pytest_generate_tests ) -from mirgecom.fluid import join_conserved, split_conserved +from mirgecom.fluid import join_conserved from mirgecom.eos import IdealSingleGas from grudge.eager import EagerDGDiscretization -def test_basic_healthcheck(actx_factory): - from mirgecom.simutil import sim_healthcheck +def test_basic_cfd_healthcheck(actx_factory): + from mirgecom.simutil import sim_cfd_healthcheck actx = actx_factory() nel_1d = 4 @@ -66,12 +66,11 @@ def test_basic_healthcheck(actx_factory): eos = IdealSingleGas() q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) - cv = split_conserved(dim, q) from mirgecom.exceptions import SimulationHealthError with pytest.raises(SimulationHealthError): - sim_healthcheck(discr, eos, q, cv) + sim_cfd_healthcheck(discr, eos, q, ncheck=1) # Let's make another very bad state (nans) mass = 1*ones @@ -80,10 +79,9 @@ def test_basic_healthcheck(actx_factory): mom = mass * velocity q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) - cv = split_conserved(dim, q) with pytest.raises(SimulationHealthError): - sim_healthcheck(discr, eos, q, cv) + sim_cfd_healthcheck(discr, eos, q, ncheck=1) # Let's make one last very bad state (inf) mass = 1*ones @@ -92,7 +90,6 @@ def test_basic_healthcheck(actx_factory): mom = mass * velocity q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) - cv = split_conserved(dim, q) with pytest.raises(SimulationHealthError): - sim_healthcheck(discr, eos, q, cv) + sim_cfd_healthcheck(discr, eos, q, ncheck=1) From b6560793ba4dd23ac3395322d5aaf50beed29256 Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Sat, 5 Jun 2021 10:58:22 -0500 Subject: [PATCH 118/385] Fix exception docs --- mirgecom/exceptions.py | 5 ++--- mirgecom/simutil.py | 6 +----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/mirgecom/exceptions.py b/mirgecom/exceptions.py index e53560b00..fc1f1d363 100644 --- a/mirgecom/exceptions.py +++ b/mirgecom/exceptions.py @@ -1,6 +1,6 @@ """Provide custom exceptions for use in callback routines. -.. autoexception:: MirgecomException +.. autoexception:: StepperCrashError .. autoexception:: SimulationHealthError """ @@ -38,8 +38,7 @@ class StepperCrashError(Exception): """ def __init__(self, message): - self.message = message - super().__init__(self.message) + super().__init__(message) class SimulationHealthError(StepperCrashError): diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 7b36b7ed7..108a1d588 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -180,7 +180,7 @@ def sim_checkpoint(discr, eos, q, exact_soln=None, def sim_cfd_healthcheck(discr, eos, q, ncheck=-1, step=0, t=0): - """Check the global health of the fluids state *q. + """Check the global health of the fluids state *q*. Determine the health of a state by inspecting for unphyiscal values of pressure and temperature. @@ -198,10 +198,6 @@ def sim_cfd_healthcheck(discr, eos, q, ncheck=-1, step=0, t=0): of species masses. ncheck: int An integer denoting the frequency interval. - - Returns - ------- - None """ from mirgecom.fluid import split_conserved import grudge.op as op From 2acdf7377238bee3b25d6dbf45b892cae5ad5b58 Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Sat, 5 Jun 2021 11:37:07 -0500 Subject: [PATCH 119/385] Expand documentation of callbacks --- mirgecom/simutil.py | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 108a1d588..3aafadbe4 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -80,7 +80,28 @@ def sim_visualization(discr, state, eos, visualizer, vizname, step=0, t=0, nviz=-1, exact_soln=None, viz_fields=None, overwrite=False, vis_timer=None): - """Visualize the simulation fields.""" + """Visualize the simulation fields. + + Write VTK output of the conserved state and and specified derived + quantities *viz_fields*. + + Parameters + ---------- + state + State array which expects at least the conserved quantities + (mass, energy, momentum) for the fluid at each point. For multi-component + fluids, the conserved quantities should include + (mass, energy, momentum, species_mass), where *species_mass* is a vector + of species masses. + eos: mirgecom.eos.GasEOS + Implementing the pressure and temperature functions for + returning pressure and temperature as a function of the state q. + visualizer: + A :class:`meshmode.discretization.visualization.Visualizer` + VTK output object. + nviz: int + An integer denoting the frequency interval. + """ from contextlib import nullcontext from mirgecom.fluid import split_conserved from mirgecom.io import make_rank_fname, make_par_fname @@ -132,7 +153,25 @@ def sim_visualization(discr, state, eos, visualizer, vizname, def sim_checkpoint(discr, eos, q, exact_soln=None, step=0, t=0, dt=0, cfl=1.0, nstatus=-1, exittol=1e-16, constant_cfl=False): - """Check simulation status, and restart.""" + """Checkpoint the simulation status. + + Checkpoints the simulation status by reporting relevant diagnostic + quantities, such as pressure/temperature. + + Parameters + ---------- + q + State array which expects at least the conserved quantities + (mass, energy, momentum) for the fluid at each point. For multi-component + fluids, the conserved quantities should include + (mass, energy, momentum, species_mass), where *species_mass* is a vector + of species masses. + eos: mirgecom.eos.GasEOS + Implementing the pressure and temperature functions for + returning pressure and temperature as a function of the state q. + nstatus: int + An integer denoting the frequency interval. + """ from mirgecom.fluid import split_conserved do_status = check_step(step=step, interval=nstatus) From 54056043484661dc0d70173400c4c34c41b8298d Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Sat, 5 Jun 2021 16:34:32 -0500 Subject: [PATCH 120/385] Simplify sim_checkpoint and modularize callbacks --- examples/autoignition-mpi.py | 25 +-- examples/lump-mpi.py | 26 +-- examples/mixture-mpi.py | 26 +-- examples/pulse-mpi.py | 26 +-- examples/scalar-lump-mpi.py | 26 +-- examples/sod-mpi.py | 26 +-- examples/vortex-mpi.py | 28 ++-- mirgecom/simutil.py | 310 ++++++++++++++++++++--------------- test/test_callbacks.py | 8 +- 9 files changed, 282 insertions(+), 219 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 5881651ea..67d10a5cc 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -41,7 +41,7 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - sim_cfd_healthcheck, + cfd_healthcheck, check_step, generate_and_distribute_mesh ) @@ -233,29 +233,32 @@ def my_rhs(t, state): + eos.get_species_source_terms(cv)) def my_checkpoint(step, t, dt, state): + cv = split_conserved(dim, state) + reaction_rates = eos.get_production_rates(cv) + viz_fields = [("reaction_rates", reaction_rates)] try: # Check the health of the simulation - sim_cfd_healthcheck(discr, eos, q=state, - ncheck=ncheck, step=step, t=t) + cfd_healthcheck(discr, eos, state, + step=step, t=t, freq=ncheck) # Perform checkpointing - sim_checkpoint(discr, eos, q=state, - exact_soln=initializer, - step=step, t=t, dt=dt, - nstatus=nstatus, + sim_checkpoint(discr, eos, state, + step=step, t=t, dt=dt, freq=nstatus, constant_cfl=constant_cfl) # Visualize - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=nviz) + step=step, t=t, freq=nviz, + viz_fields=viz_fields) except StepperCrashError as err: # Log crash error message if rank == 0: logger.info(str(err)) logger.info("Visualizing crashed state ...") # Write out crashed field - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=1) + step=step, t=t, freq=1, + viz_fields=viz_fields) raise err return state diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 44130fa1c..7b24287f9 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -41,7 +41,8 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - sim_cfd_healthcheck, + cfd_healthcheck, + compare_with_analytic_solution, generate_and_distribute_mesh ) from mirgecom.exceptions import StepperCrashError @@ -136,27 +137,30 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): try: # Check the health of the simulation - sim_cfd_healthcheck(discr, eos, q=state, - ncheck=ncheck, step=step, t=t) + cfd_healthcheck(discr, eos, state, + step=step, t=t, freq=ncheck) # Perform checkpointing - sim_checkpoint(discr, eos, q=state, - exact_soln=initializer, - step=step, t=t, dt=dt, - nstatus=nstatus, exittol=exittol, + sim_checkpoint(discr, eos, state, + step=step, t=t, dt=dt, freq=nstatus, constant_cfl=constant_cfl) + # Compare with analytic result + compare_with_analytic_solution(discr, eos, state, + exact_soln=initializer, + step=step, t=t, freq=nstatus, + exittol=exittol) # Visualize - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=nviz) + step=step, t=t, freq=nviz) except StepperCrashError as err: # Log crash error message if rank == 0: logger.info(str(err)) logger.info("Visualizing crashed state ...") # Write out crashed field - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=1) + step=step, t=t, freq=1) raise err return state diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 3b5bc921e..33e3c2374 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -41,7 +41,8 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - sim_cfd_healthcheck, + cfd_healthcheck, + compare_with_analytic_solution, generate_and_distribute_mesh ) from mirgecom.exceptions import StepperCrashError @@ -156,27 +157,30 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): try: # Check the health of the simulation - sim_cfd_healthcheck(discr, eos, q=state, - ncheck=ncheck, step=step, t=t) + cfd_healthcheck(discr, eos, state, + step=step, t=t, freq=ncheck) # Perform checkpointing - sim_checkpoint(discr, eos, q=state, - exact_soln=initializer, - step=step, t=t, dt=dt, - nstatus=nstatus, exittol=exittol, + sim_checkpoint(discr, eos, state, + step=step, t=t, dt=dt, freq=nstatus, constant_cfl=constant_cfl) + # Compare with analytic result + compare_with_analytic_solution(discr, eos, state, + exact_soln=initializer, + step=step, t=t, freq=nstatus, + exittol=exittol) # Visualize - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=nviz) + step=step, t=t, freq=nviz) except StepperCrashError as err: # Log crash error message if rank == 0: logger.info(str(err)) logger.info("Visualizing crashed state ...") # Write out crashed field - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=1) + step=step, t=t, freq=1) raise err return state diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 74f1e0366..114dce3c4 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -42,7 +42,8 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - sim_cfd_healthcheck, + cfd_healthcheck, + compare_with_analytic_solution, generate_and_distribute_mesh ) from mirgecom.exceptions import StepperCrashError @@ -158,27 +159,30 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): try: # Check the health of the simulation - sim_cfd_healthcheck(discr, eos, q=state, - ncheck=ncheck, step=step, t=t) + cfd_healthcheck(discr, eos, state, + step=step, t=t, freq=ncheck) # Perform checkpointing - sim_checkpoint(discr, eos, q=state, - exact_soln=initializer, - step=step, t=t, dt=dt, - nstatus=nstatus, exittol=exittol, + sim_checkpoint(discr, eos, state, + step=step, t=t, dt=dt, freq=nstatus, constant_cfl=constant_cfl) + # Compare with analytic result + compare_with_analytic_solution(discr, eos, state, + exact_soln=initializer, + step=step, t=t, freq=nstatus, + exittol=exittol) # Visualize - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=nviz) + step=step, t=t, freq=nviz) except StepperCrashError as err: # Log crash error message if rank == 0: logger.info(str(err)) logger.info("Visualizing crashed state ...") # Write out crashed field - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=1) + step=step, t=t, freq=1) raise err return state diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index a8dc1696b..2fb8bac61 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -42,7 +42,8 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - sim_cfd_healthcheck, + cfd_healthcheck, + compare_with_analytic_solution, generate_and_distribute_mesh ) from mirgecom.exceptions import StepperCrashError @@ -146,27 +147,30 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): try: # Check the health of the simulation - sim_cfd_healthcheck(discr, eos, q=state, - ncheck=ncheck, step=step, t=t) + cfd_healthcheck(discr, eos, state, + step=step, t=t, freq=ncheck) # Perform checkpointing - sim_checkpoint(discr, eos, q=state, - exact_soln=initializer, - step=step, t=t, dt=dt, - nstatus=nstatus, exittol=exittol, + sim_checkpoint(discr, eos, state, + step=step, t=t, dt=dt, freq=nstatus, constant_cfl=constant_cfl) + # Compare with analytic result + compare_with_analytic_solution(discr, eos, state, + exact_soln=initializer, + step=step, t=t, freq=nstatus, + exittol=exittol) # Visualize - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=nviz) + step=step, t=t, freq=nviz) except StepperCrashError as err: # Log crash error message if rank == 0: logger.info(str(err)) logger.info("Visualizing crashed state ...") # Write out crashed field - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=1) + step=step, t=t, freq=1) raise err return state diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 84fb47a4d..3cd2e8273 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -40,7 +40,8 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - sim_cfd_healthcheck, + cfd_healthcheck, + compare_with_analytic_solution, generate_and_distribute_mesh ) from mirgecom.exceptions import StepperCrashError @@ -135,27 +136,30 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): try: # Check the health of the simulation - sim_cfd_healthcheck(discr, eos, q=state, - ncheck=ncheck, step=step, t=t) + cfd_healthcheck(discr, eos, state, + step=step, t=t, freq=ncheck) # Perform checkpointing - sim_checkpoint(discr, eos, q=state, - exact_soln=initializer, - step=step, t=t, dt=dt, - nstatus=nstatus, exittol=exittol, + sim_checkpoint(discr, eos, state, + step=step, t=t, dt=dt, freq=nstatus, constant_cfl=constant_cfl) + # Compare with analytic result + compare_with_analytic_solution(discr, eos, state, + exact_soln=initializer, + step=step, t=t, freq=nstatus, + exittol=exittol) # Visualize - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=nviz) + step=step, t=t, freq=nviz) except StepperCrashError as err: # Log crash error message if rank == 0: logger.info(str(err)) logger.info("Visualizing crashed state ...") # Write out crashed field - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=1) + step=step, t=t, freq=1) raise err return state diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 5014582c2..a6da18065 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -42,7 +42,8 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - sim_cfd_healthcheck, + cfd_healthcheck, + compare_with_analytic_solution, generate_and_distribute_mesh ) from mirgecom.exceptions import StepperCrashError @@ -182,29 +183,30 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): try: # Check the health of the simulation - sim_cfd_healthcheck(discr, eos, q=state, - ncheck=ncheck, step=step, t=t) + cfd_healthcheck(discr, eos, state, + step=step, t=t, freq=ncheck) # Perform checkpointing - sim_checkpoint(discr, eos, q=state, - exact_soln=initializer, - step=step, t=t, dt=dt, - nstatus=nstatus, exittol=exittol, + sim_checkpoint(discr, eos, state, + step=step, t=t, dt=dt, freq=nstatus, constant_cfl=constant_cfl) + # Compare with analytic result + compare_with_analytic_solution(discr, eos, state, + exact_soln=initializer, + step=step, t=t, freq=nstatus, + exittol=exittol) # Visualize - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=nviz, - vis_timer=vis_timer) + step=step, t=t, freq=nviz) except StepperCrashError as err: # Log crash error message if rank == 0: logger.info(str(err)) logger.info("Visualizing crashed state ...") # Write out crashed field - sim_visualization(discr, state, eos, + sim_visualization(discr, eos, state, visualizer, vizname=casename, - step=step, t=t, nviz=1, - vis_timer=vis_timer) + step=step, t=t, freq=1) raise err return state diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 3aafadbe4..18c69df8d 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1,10 +1,22 @@ """Provide some utilities for building simulation applications. +General utilities +----------------- + .. autofunction:: check_step .. autofunction:: inviscid_sim_timestep + +Diagnostic callbacks +-------------------- + .. autofunction:: sim_visualization .. autofunction:: sim_checkpoint -.. autofunction:: sim_cfd_healthcheck +.. autofunction:: cfd_healthcheck +.. autofunction:: compare_with_analytic_solution + +Mesh utilities +-------------- + .. autofunction:: generate_and_distribute_mesh """ @@ -76,8 +88,8 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, return mydt -def sim_visualization(discr, state, eos, visualizer, vizname, - step=0, t=0, nviz=-1, +def sim_visualization(discr, eos, state, visualizer, vizname, + step=0, t=0, freq=-1, exact_soln=None, viz_fields=None, overwrite=False, vis_timer=None): """Visualize the simulation fields. @@ -87,72 +99,70 @@ def sim_visualization(discr, state, eos, visualizer, vizname, Parameters ---------- + eos: mirgecom.eos.GasEOS + Implementing the pressure and temperature functions for + returning pressure and temperature as a function of the state. state State array which expects at least the conserved quantities (mass, energy, momentum) for the fluid at each point. For multi-component fluids, the conserved quantities should include (mass, energy, momentum, species_mass), where *species_mass* is a vector of species masses. - eos: mirgecom.eos.GasEOS - Implementing the pressure and temperature functions for - returning pressure and temperature as a function of the state q. visualizer: A :class:`meshmode.discretization.visualization.Visualizer` VTK output object. - nviz: int - An integer denoting the frequency interval. + freq: int + An integer denoting the step frequency. """ - from contextlib import nullcontext - from mirgecom.fluid import split_conserved - from mirgecom.io import make_rank_fname, make_par_fname - - if not check_step(step=step, interval=nviz): - return - - cv = split_conserved(discr.dim, state) - dependent_vars = eos.dependent_vars(cv) - - io_fields = [ - ("cv", cv), - ("dv", dependent_vars) - ] - if exact_soln is not None: - actx = cv.mass.array_context - nodes = thaw(actx, discr.nodes()) - expected_state = exact_soln(x_vec=nodes, t=t, eos=eos) - exact_list = [ - ("exact_soln", expected_state), - ] - io_fields.extend(exact_list) + if check_step(step=step, interval=freq): - if viz_fields is not None: - io_fields.extend(viz_fields) + from contextlib import nullcontext + from mirgecom.fluid import split_conserved + from mirgecom.io import make_rank_fname, make_par_fname - comm = discr.mpi_communicator - rank = 0 - if comm is not None: - rank = comm.Get_rank() + cv = split_conserved(discr.dim, state) + dependent_vars = eos.dependent_vars(cv) - rank_fn = make_rank_fname(basename=vizname, rank=rank, step=step, t=t) - - if vis_timer: - ctm = vis_timer.start_sub_timer() - else: - ctm = nullcontext() - - with ctm: - visualizer.write_parallel_vtk_file( - comm, rank_fn, io_fields, - overwrite=overwrite, - par_manifest_filename=make_par_fname( - basename=vizname, step=step, t=t + io_fields = [ + ("cv", cv), + ("dv", dependent_vars) + ] + if exact_soln is not None: + actx = cv.mass.array_context + nodes = thaw(actx, discr.nodes()) + expected_state = exact_soln(x_vec=nodes, t=t, eos=eos) + exact_list = [ + ("exact_soln", expected_state), + ] + io_fields.extend(exact_list) + + if viz_fields is not None: + io_fields.extend(viz_fields) + + comm = discr.mpi_communicator + rank = 0 + if comm is not None: + rank = comm.Get_rank() + + rank_fn = make_rank_fname(basename=vizname, rank=rank, step=step, t=t) + + if vis_timer: + ctm = vis_timer.start_sub_timer() + else: + ctm = nullcontext() + + with ctm: + visualizer.write_parallel_vtk_file( + comm, rank_fn, io_fields, + overwrite=overwrite, + par_manifest_filename=make_par_fname( + basename=vizname, step=step, t=t + ) ) - ) -def sim_checkpoint(discr, eos, q, exact_soln=None, - step=0, t=0, dt=0, cfl=1.0, nstatus=-1, exittol=1e-16, - constant_cfl=False): +def sim_checkpoint(discr, eos, state, step=0, t=0, dt=0, + cfl=1.0, freq=-1, constant_cfl=False): """Checkpoint the simulation status. Checkpoints the simulation status by reporting relevant diagnostic @@ -160,66 +170,42 @@ def sim_checkpoint(discr, eos, q, exact_soln=None, Parameters ---------- - q + eos: mirgecom.eos.GasEOS + Implementing the pressure and temperature functions for + returning pressure and temperature as a function of the state. + state State array which expects at least the conserved quantities (mass, energy, momentum) for the fluid at each point. For multi-component fluids, the conserved quantities should include (mass, energy, momentum, species_mass), where *species_mass* is a vector of species masses. - eos: mirgecom.eos.GasEOS - Implementing the pressure and temperature functions for - returning pressure and temperature as a function of the state q. - nstatus: int - An integer denoting the frequency interval. + freq: int + An integer denoting the step frequency. """ - from mirgecom.fluid import split_conserved - - do_status = check_step(step=step, interval=nstatus) - if do_status is False: - return + if check_step(step=step, interval=freq): - cv = split_conserved(discr.dim, q) - dependent_vars = eos.dependent_vars(cv) + from mirgecom.fluid import split_conserved - maxerr = 0.0 - if exact_soln is not None: - actx = cv.mass.array_context - nodes = thaw(actx, discr.nodes()) - expected_state = exact_soln(x_vec=nodes, t=t, eos=eos) - exp_resid = q - expected_state - err_norms = [discr.norm(v, np.inf) for v in exp_resid] - maxerr = max(err_norms) - - comm = discr.mpi_communicator - rank = 0 - if comm is not None: - rank = comm.Get_rank() - - if do_status is True: # if constant_cfl is False: - # current_cfl = get_inviscid_cfl(discr=discr, q=q, + # current_cfl = get_inviscid_cfl(discr=discr, q=state, # eos=eos, dt=dt) - statusmesg = make_status_message(discr=discr, t=t, step=step, dt=dt, - cfl=cfl, dependent_vars=dependent_vars) - if exact_soln is not None: - statusmesg += ( - "\n------- errors=" - + ", ".join("%.3g" % en for en in err_norms)) - if rank == 0: - logger.info(statusmesg) + comm = discr.mpi_communicator + rank = 0 + if comm is not None: + rank = comm.Get_rank() - if maxerr > exittol: - raise SimulationHealthError( - message=( - "Simulation exited abnormally; " - "solution doesn't agree with analytic result." - ) - ) + cv = split_conserved(discr.dim, state) + dependent_vars = eos.dependent_vars(cv) + msg = make_status_message(discr=discr, + t=t, step=step, dt=dt, cfl=cfl, + dependent_vars=dependent_vars) + if rank == 0: + logger.info(msg) -def sim_cfd_healthcheck(discr, eos, q, ncheck=-1, step=0, t=0): - """Check the global health of the fluids state *q*. +def cfd_healthcheck(discr, eos, state, step=0, t=0, freq=-1): + """Check the global health of the fluids state. Determine the health of a state by inspecting for unphyiscal values of pressure and temperature. @@ -228,57 +214,109 @@ def sim_cfd_healthcheck(discr, eos, q, ncheck=-1, step=0, t=0): ---------- eos: mirgecom.eos.GasEOS Implementing the pressure and temperature functions for - returning pressure and temperature as a function of the state q. - q + returning pressure and temperature as a function of the state. + state State array which expects at least the conserved quantities (mass, energy, momentum) for the fluid at each point. For multi-component fluids, the conserved quantities should include (mass, energy, momentum, species_mass), where *species_mass* is a vector of species masses. - ncheck: int - An integer denoting the frequency interval. + freq: int + An integer denoting the step frequency. """ - from mirgecom.fluid import split_conserved - import grudge.op as op + if check_step(step=step, interval=freq): + + from mirgecom.fluid import split_conserved + import grudge.op as op + + # NOTE: Derived quantities are functions of the conserved variables. + # Therefore is it sufficient to check for unphysical values of + # temperature and pressure. + cv = split_conserved(discr.dim, state) + dependent_vars = eos.dependent_vars(cv) + pressure = dependent_vars.pressure + temperature = dependent_vars.temperature + + # Check for NaN + if (np.isnan(op.nodal_sum_loc(discr, "vol", pressure)) + or np.isnan(op.nodal_sum_loc(discr, "vol", temperature))): + raise SimulationHealthError( + message=("Simulation exited abnormally; detected a NaN.") + ) - if not check_step(step=step, interval=ncheck): - return + # Check for non-positivity + if (op.nodal_min_loc(discr, "vol", pressure) < 0 + or op.nodal_min_loc(discr, "vol", temperature) < 0): + raise SimulationHealthError( + message=("Simulation exited abnormally; " + "found non-positive values for pressure or temperature.") + ) - cv = split_conserved(discr.dim, q) + # Check for blow-up + if (op.nodal_sum_loc(discr, "vol", pressure) == np.inf + or op.nodal_sum_loc(discr, "vol", temperature) == np.inf): + raise SimulationHealthError( + message=("Simulation exited abnormally; " + "derived quantities are not finite.") + ) - # NOTE: Derived quantities are functions of the conserved variables. - # Therefore is it sufficient to check for unphysical values of - # temperature and pressure. - dependent_vars = eos.dependent_vars(cv) - # Check for NaN - if (np.isnan(op.nodal_sum_loc(discr, "vol", dependent_vars.pressure)) - or np.isnan(op.nodal_sum_loc(discr, "vol", dependent_vars.temperature))): - raise SimulationHealthError( - message=( - "Simulation exited abnormally; detected a NaN." - ) - ) +def compare_with_analytic_solution(discr, eos, state, exact_soln, + step=0, t=0, freq=-1, exittol=None): + """Compute the infinite norm of the problem residual. - # Check for non-positivity - if (op.nodal_min_loc(discr, "vol", dependent_vars.pressure) < 0 - or op.nodal_min_loc(discr, "vol", dependent_vars.temperature) < 0): - raise SimulationHealthError( - message=( - "Simulation exited abnormally; " - "found non-positive values for pressure or temperature." - ) + Computes the infinite norm of the residual with respect to a specified + exact solution *exact_soln*. If the error is larger than *exittol*, + raises a :class:`mirgecom.exceptions.SimulationHealthError`. + + Parameters + ---------- + eos: mirgecom.eos.GasEOS + Implementing the pressure and temperature functions for + returning pressure and temperature as a function of the state. + state + State array which expects at least the conserved quantities + (mass, energy, momentum) for the fluid at each point. For multi-component + fluids, the conserved quantities should include + (mass, energy, momentum, species_mass), where *species_mass* is a vector + of species masses. + exact_soln: + A callable for the exact solution with signature: + ``exact_soln(x_vec, t, eos)`` where `x_vec` are the nodes, + `t` is time, and `eos` is a :class:`mirgecom.eos.GasEOS`. + freq: int + An integer denoting the step frequency. + """ + if check_step(step=step, interval=freq): + + if exittol is None: + exittol = 1e-16 + + actx = discr._setup_actx + nodes = thaw(actx, discr.nodes()) + expected_state = exact_soln(x_vec=nodes, t=t, eos=eos) + exp_resid = state - expected_state + norms = [discr.norm(v, np.inf) for v in exp_resid] + + comm = discr.mpi_communicator + rank = 0 + if comm is not None: + rank = comm.Get_rank() + + statusmesg = ( + f"Errors: {step=} {t=}\n" + f"------- errors = " + + ", ".join("%.3g" % err_norm for err_norm in norms) ) - # Check for blow-up - if (op.nodal_sum_loc(discr, "vol", dependent_vars.pressure) == np.inf - or op.nodal_sum_loc(discr, "vol", dependent_vars.temperature) == np.inf): - raise SimulationHealthError( - message=( - "Simulation exited abnormally; " - "derived quantities are not finite." + if rank == 0: + logger.info(statusmesg) + + if max(norms) > exittol: + raise SimulationHealthError( + message=("Simulation exited abnormally; " + "solution doesn't agree with analytic result.") ) - ) def generate_and_distribute_mesh(comm, generate_mesh): diff --git a/test/test_callbacks.py b/test/test_callbacks.py index 72d9ff82a..caad9542b 100644 --- a/test/test_callbacks.py +++ b/test/test_callbacks.py @@ -40,7 +40,7 @@ def test_basic_cfd_healthcheck(actx_factory): - from mirgecom.simutil import sim_cfd_healthcheck + from mirgecom.simutil import cfd_healthcheck actx = actx_factory() nel_1d = 4 @@ -70,7 +70,7 @@ def test_basic_cfd_healthcheck(actx_factory): from mirgecom.exceptions import SimulationHealthError with pytest.raises(SimulationHealthError): - sim_cfd_healthcheck(discr, eos, q, ncheck=1) + cfd_healthcheck(discr, eos, q, freq=1) # Let's make another very bad state (nans) mass = 1*ones @@ -81,7 +81,7 @@ def test_basic_cfd_healthcheck(actx_factory): q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) with pytest.raises(SimulationHealthError): - sim_cfd_healthcheck(discr, eos, q, ncheck=1) + cfd_healthcheck(discr, eos, q, freq=1) # Let's make one last very bad state (inf) mass = 1*ones @@ -92,4 +92,4 @@ def test_basic_cfd_healthcheck(actx_factory): q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) with pytest.raises(SimulationHealthError): - sim_cfd_healthcheck(discr, eos, q, ncheck=1) + cfd_healthcheck(discr, eos, q, freq=1) From ca7c37b608679dbca6f25ce3cde362e8880f69a7 Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Sun, 6 Jun 2021 00:07:59 -0500 Subject: [PATCH 121/385] Update steppers --- mirgecom/steppers.py | 72 +++++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 0ca3af0b6..b069f38e5 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -5,7 +5,7 @@ """ __copyright__ = """ -Copyright (C) 2020 University of Illinois Board of Trustees +Copyright (C) 2020-21 University of Illinois Board of Trustees """ __license__ = """ @@ -32,11 +32,12 @@ from mirgecom.logging_quantities import set_sim_state -def advance_state(rhs, timestepper, get_timestep, - state, t_final, t=0.0, istep=0, - pre_step_callback=None, - post_step_callback=None, - logmgr=None, eos=None, dim=None): +def _advance_state_stepper_func(rhs, timestepper, get_timestep, + state, t_final, + t=0.0, istep=0, + pre_step_callback=None, + post_step_callback=None, + logmgr=None, eos=None, dim=None): """Advance state from some time (t) to some time (t_final). Parameters @@ -47,7 +48,8 @@ def advance_state(rhs, timestepper, get_timestep, a call with signature ``rhs(t, state)``. timestepper Function that advances the state from t=time to t=(time+dt), and - returns the advanced state. + returns the advanced state. Has a call with signature + ``timestepper(state, t, dt, rhs)``. get_timestep Function that should return dt for the next step. This interface allows user-defined adaptive timestepping. A negative return value indicated that @@ -110,9 +112,13 @@ def advance_state(rhs, timestepper, get_timestep, return istep, t, state -def _advance_state_leap(rhs, timestepper, checkpoint, get_timestep, - state, t_final, component_id="state", t=0.0, istep=0, - logmgr=None, eos=None, dim=None): +def _advance_state_leap(rhs, timestepper, get_timestep, + state, t_final, + component_id="state", + t=0.0, istep=0, + pre_step_callback=None, + post_step_callback=None, + logmgr=None, eos=None, dim=None): """Advance state from some time *t* to some time *t_final* using :mod:`leap`. Parameters @@ -123,10 +129,6 @@ def _advance_state_leap(rhs, timestepper, checkpoint, get_timestep, a call with signature ``rhs(t, state)``. timestepper An instance of :class:`leap.MethodBuilder`. - checkpoint - Function is user-defined and can be used to preform simulation status - reporting, viz, and restart i/o. A non-zero return code from this function - indicates that this function should stop gracefully. get_timestep Function that should return dt for the next step. This interface allows user-defined adaptive timestepping. A negative return value indicated that @@ -192,6 +194,7 @@ def _advance_state_leap(rhs, timestepper, checkpoint, get_timestep, t=t, dt=dt) istep += 1 + if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) @@ -239,9 +242,12 @@ def generate_singlerate_leap_advancer(timestepper, component_id, rhs, t, dt, return stepper_cls -def advance_state(rhs, timestepper, checkpoint, get_timestep, state, t_final, - component_id="state", t=0.0, istep=0, logmgr=None, - eos=None, dim=None): +def advance_state(rhs, timestepper, get_timestep, state, t_final, + component_id="state", + t=0.0, istep=0, + pre_step_callback=None, + post_step_callback=None, + logmgr=None, eos=None, dim=None): """Determine what stepper we're using and advance the state from (t) to (t_final). Parameters @@ -258,10 +264,6 @@ def advance_state(rhs, timestepper, checkpoint, get_timestep, state, t_final, responsible for generating timestepper code from the method instructions before using it, as well as providing context in the form of the state to be integrated, the initial time and timestep, and the RHS function. - checkpoint - Function is user-defined and can be used to preform simulation status - reporting, viz, and restart i/o. A non-zero return code from this function - indicates that this function should stop gracefully. component_id State id (required input for leap method generation) get_timestep @@ -308,17 +310,25 @@ def advance_state(rhs, timestepper, checkpoint, get_timestep, state, t_final, if leap_timestepper: (current_step, current_t, current_state) = \ - _advance_state_leap(rhs=rhs, timestepper=timestepper, - checkpoint=checkpoint, - get_timestep=get_timestep, state=state, - t=t, t_final=t_final, component_id=component_id, - istep=istep, logmgr=logmgr, eos=eos, dim=dim) + _advance_state_leap( + rhs=rhs, timestepper=timestepper, + get_timestep=get_timestep, state=state, + t=t, t_final=t_final, + pre_step_callback=pre_step_callback, + post_step_callback=post_step_callback, + component_id=component_id, + istep=istep, logmgr=logmgr, eos=eos, dim=dim + ) else: (current_step, current_t, current_state) = \ - _advance_state_stepper_func(rhs=rhs, timestepper=timestepper, - checkpoint=checkpoint, - get_timestep=get_timestep, state=state, - t=t, t_final=t_final, istep=istep, - logmgr=logmgr, eos=eos, dim=dim) + _advance_state_stepper_func( + rhs=rhs, timestepper=timestepper, + get_timestep=get_timestep, state=state, + t=t, t_final=t_final, + pre_step_callback=pre_step_callback, + post_step_callback=post_step_callback, + istep=istep, + logmgr=logmgr, eos=eos, dim=dim + ) return current_step, current_t, current_state From da65cd2e277209f6da18fb54286a650c25d294dc Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Sun, 6 Jun 2021 00:27:03 -0500 Subject: [PATCH 122/385] Write short test for comparison callback --- test/test_callbacks.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/test_callbacks.py b/test/test_callbacks.py index caad9542b..49db3eb63 100644 --- a/test/test_callbacks.py +++ b/test/test_callbacks.py @@ -93,3 +93,36 @@ def test_basic_cfd_healthcheck(actx_factory): with pytest.raises(SimulationHealthError): cfd_healthcheck(discr, eos, q, freq=1) + + +def test_analytic_comparison(actx_factory): + from mirgecom.initializers import Vortex2D + from mirgecom.simutil import compare_with_analytic_solution + + actx = actx_factory() + nel_1d = 4 + dim = 2 + + from meshmode.mesh.generation import generate_regular_rect_mesh + + mesh = generate_regular_rect_mesh( + a=(1.0,) * dim, b=(2.0,) * dim, nelements_per_axis=(nel_1d,) * dim + ) + + order = 2 + discr = EagerDGDiscretization(actx, mesh, order=order) + nodes = thaw(discr.nodes(), actx) + zeros = discr.zeros(actx) + ones = zeros + 1.0 + eos = IdealSingleGas() + mass = ones + energy = ones + velocity = 2 * nodes + mom = mass * velocity + + q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) + + from mirgecom.exceptions import SimulationHealthError + + with pytest.raises(SimulationHealthError): + compare_with_analytic_solution(discr, eos, q, Vortex2D(), freq=1) From 63c9a8068e0c152cf8fafdae3af2083cbb5d8972 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 7 Jun 2021 10:23:04 -0500 Subject: [PATCH 123/385] Update to new dt API since merge with #368 --- examples/nsmix-mpi.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/nsmix-mpi.py b/examples/nsmix-mpi.py index 7797bceaa..08b223287 100644 --- a/examples/nsmix-mpi.py +++ b/examples/nsmix-mpi.py @@ -238,8 +238,9 @@ def get_timestep(state): next_dt = current_dt t_end = t_final if constant_cfl is True: - inviscid_dt = get_inviscid_timestep(discr=discr, eos=eos, - cfl=current_cfl, q=state) + inviscid_dt = ( + current_cfl * get_inviscid_timestep(discr=discr, eos=eos, q=state) + ) viscous_dt = get_viscous_timestep(discr=discr, eos=eos, transport_model=transport_model, cfl=current_cfl, q=state) From ebb3f9d27a007da05f017e09898d8ce719f0e86a Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 7 Jun 2021 11:20:22 -0500 Subject: [PATCH 124/385] Fix "indento" that makes doublemach hang at the end! --- examples/doublemach-mpi.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 901bbadd1..df513336d 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -250,12 +250,13 @@ def my_checkpoint(step, t, dt, state): # if current_t != checkpoint_t: if rank == 0: logger.info("Checkpointing final state ...") - my_checkpoint( - current_step, - t=current_t, - dt=(current_t - checkpoint_t), - state=current_state, - ) + + my_checkpoint( + current_step, + t=current_t, + dt=(current_t - checkpoint_t), + state=current_state, + ) if current_t - t_final < 0: raise ValueError("Simulation exited abnormally") From 1bb8357332d99431f52bd7a894dba8e7f0a98e16 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 7 Jun 2021 11:21:44 -0500 Subject: [PATCH 125/385] Fix "indento" that makes doublemach hang at the end! --- examples/doublemach-mpi.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 901bbadd1..356318622 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -250,12 +250,12 @@ def my_checkpoint(step, t, dt, state): # if current_t != checkpoint_t: if rank == 0: logger.info("Checkpointing final state ...") - my_checkpoint( - current_step, - t=current_t, - dt=(current_t - checkpoint_t), - state=current_state, - ) + my_checkpoint( + current_step, + t=current_t, + dt=(current_t - checkpoint_t), + state=current_state, + ) if current_t - t_final < 0: raise ValueError("Simulation exited abnormally") From 2523fbdfa581c0c86f0e4d21691aa345a64d185b Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Mon, 7 Jun 2021 16:47:37 -0500 Subject: [PATCH 126/385] Renaming: {cfd_healthcheck,StepperCrashError} -> {sim_healthcheck,SynchronizedError} --- examples/autoignition-mpi.py | 8 ++++---- examples/lump-mpi.py | 8 ++++---- examples/mixture-mpi.py | 8 ++++---- examples/pulse-mpi.py | 8 ++++---- examples/scalar-lump-mpi.py | 8 ++++---- examples/sod-mpi.py | 8 ++++---- examples/vortex-mpi.py | 8 ++++---- mirgecom/exceptions.py | 10 +++++----- mirgecom/simutil.py | 4 ++-- test/test_callbacks.py | 8 ++++---- 10 files changed, 39 insertions(+), 39 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 67d10a5cc..6c65d28ba 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -41,11 +41,11 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - cfd_healthcheck, + sim_healthcheck, check_step, generate_and_distribute_mesh ) -from mirgecom.exceptions import StepperCrashError +from mirgecom.exceptions import SynchronizedError from mirgecom.fluid import split_conserved from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -238,7 +238,7 @@ def my_checkpoint(step, t, dt, state): viz_fields = [("reaction_rates", reaction_rates)] try: # Check the health of the simulation - cfd_healthcheck(discr, eos, state, + sim_healthcheck(discr, eos, state, step=step, t=t, freq=ncheck) # Perform checkpointing sim_checkpoint(discr, eos, state, @@ -249,7 +249,7 @@ def my_checkpoint(step, t, dt, state): visualizer, vizname=casename, step=step, t=t, freq=nviz, viz_fields=viz_fields) - except StepperCrashError as err: + except SynchronizedError as err: # Log crash error message if rank == 0: logger.info(str(err)) diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 7b24287f9..2e1e9cb88 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -41,11 +41,11 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - cfd_healthcheck, + sim_healthcheck, compare_with_analytic_solution, generate_and_distribute_mesh ) -from mirgecom.exceptions import StepperCrashError +from mirgecom.exceptions import SynchronizedError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -137,7 +137,7 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): try: # Check the health of the simulation - cfd_healthcheck(discr, eos, state, + sim_healthcheck(discr, eos, state, step=step, t=t, freq=ncheck) # Perform checkpointing sim_checkpoint(discr, eos, state, @@ -152,7 +152,7 @@ def my_checkpoint(step, t, dt, state): sim_visualization(discr, eos, state, visualizer, vizname=casename, step=step, t=t, freq=nviz) - except StepperCrashError as err: + except SynchronizedError as err: # Log crash error message if rank == 0: logger.info(str(err)) diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 33e3c2374..871897c53 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -41,11 +41,11 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - cfd_healthcheck, + sim_healthcheck, compare_with_analytic_solution, generate_and_distribute_mesh ) -from mirgecom.exceptions import StepperCrashError +from mirgecom.exceptions import SynchronizedError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -157,7 +157,7 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): try: # Check the health of the simulation - cfd_healthcheck(discr, eos, state, + sim_healthcheck(discr, eos, state, step=step, t=t, freq=ncheck) # Perform checkpointing sim_checkpoint(discr, eos, state, @@ -172,7 +172,7 @@ def my_checkpoint(step, t, dt, state): sim_visualization(discr, eos, state, visualizer, vizname=casename, step=step, t=t, freq=nviz) - except StepperCrashError as err: + except SynchronizedError as err: # Log crash error message if rank == 0: logger.info(str(err)) diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 114dce3c4..54012c178 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -42,11 +42,11 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - cfd_healthcheck, + sim_healthcheck, compare_with_analytic_solution, generate_and_distribute_mesh ) -from mirgecom.exceptions import StepperCrashError +from mirgecom.exceptions import SynchronizedError from mirgecom.io import make_init_message from mirgecom.integrators import rk4_step @@ -159,7 +159,7 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): try: # Check the health of the simulation - cfd_healthcheck(discr, eos, state, + sim_healthcheck(discr, eos, state, step=step, t=t, freq=ncheck) # Perform checkpointing sim_checkpoint(discr, eos, state, @@ -174,7 +174,7 @@ def my_checkpoint(step, t, dt, state): sim_visualization(discr, eos, state, visualizer, vizname=casename, step=step, t=t, freq=nviz) - except StepperCrashError as err: + except SynchronizedError as err: # Log crash error message if rank == 0: logger.info(str(err)) diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 2fb8bac61..be0586937 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -42,11 +42,11 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - cfd_healthcheck, + sim_healthcheck, compare_with_analytic_solution, generate_and_distribute_mesh ) -from mirgecom.exceptions import StepperCrashError +from mirgecom.exceptions import SynchronizedError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -147,7 +147,7 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): try: # Check the health of the simulation - cfd_healthcheck(discr, eos, state, + sim_healthcheck(discr, eos, state, step=step, t=t, freq=ncheck) # Perform checkpointing sim_checkpoint(discr, eos, state, @@ -162,7 +162,7 @@ def my_checkpoint(step, t, dt, state): sim_visualization(discr, eos, state, visualizer, vizname=casename, step=step, t=t, freq=nviz) - except StepperCrashError as err: + except SynchronizedError as err: # Log crash error message if rank == 0: logger.info(str(err)) diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 3cd2e8273..59d2415e4 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -40,11 +40,11 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - cfd_healthcheck, + sim_healthcheck, compare_with_analytic_solution, generate_and_distribute_mesh ) -from mirgecom.exceptions import StepperCrashError +from mirgecom.exceptions import SynchronizedError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -136,7 +136,7 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): try: # Check the health of the simulation - cfd_healthcheck(discr, eos, state, + sim_healthcheck(discr, eos, state, step=step, t=t, freq=ncheck) # Perform checkpointing sim_checkpoint(discr, eos, state, @@ -151,7 +151,7 @@ def my_checkpoint(step, t, dt, state): sim_visualization(discr, eos, state, visualizer, vizname=casename, step=step, t=t, freq=nviz) - except StepperCrashError as err: + except SynchronizedError as err: # Log crash error message if rank == 0: logger.info(str(err)) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index a6da18065..9fbe01a7a 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -42,11 +42,11 @@ inviscid_sim_timestep, sim_visualization, sim_checkpoint, - cfd_healthcheck, + sim_healthcheck, compare_with_analytic_solution, generate_and_distribute_mesh ) -from mirgecom.exceptions import StepperCrashError +from mirgecom.exceptions import SynchronizedError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -183,7 +183,7 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): try: # Check the health of the simulation - cfd_healthcheck(discr, eos, state, + sim_healthcheck(discr, eos, state, step=step, t=t, freq=ncheck) # Perform checkpointing sim_checkpoint(discr, eos, state, @@ -198,7 +198,7 @@ def my_checkpoint(step, t, dt, state): sim_visualization(discr, eos, state, visualizer, vizname=casename, step=step, t=t, freq=nviz) - except StepperCrashError as err: + except SynchronizedError as err: # Log crash error message if rank == 0: logger.info(str(err)) diff --git a/mirgecom/exceptions.py b/mirgecom/exceptions.py index fc1f1d363..8d43c81a1 100644 --- a/mirgecom/exceptions.py +++ b/mirgecom/exceptions.py @@ -1,6 +1,6 @@ """Provide custom exceptions for use in callback routines. -.. autoexception:: StepperCrashError +.. autoexception:: SynchronizedError .. autoexception:: SimulationHealthError """ @@ -29,17 +29,17 @@ """ -class StepperCrashError(Exception): - """Exception base class for simulation exceptions. +class SynchronizedError(Exception): + """Exception base class which must be globally synchronized. .. attribute:: message - A :class:`str` describing the message for the exception. + A :class:`str` describing the message for the global exception. """ def __init__(self, message): super().__init__(message) -class SimulationHealthError(StepperCrashError): +class SimulationHealthError(SynchronizedError): """Exception class for an unphysical simulation.""" diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 18c69df8d..7e5d52dbb 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -11,7 +11,7 @@ .. autofunction:: sim_visualization .. autofunction:: sim_checkpoint -.. autofunction:: cfd_healthcheck +.. autofunction:: sim_healthcheck .. autofunction:: compare_with_analytic_solution Mesh utilities @@ -204,7 +204,7 @@ def sim_checkpoint(discr, eos, state, step=0, t=0, dt=0, logger.info(msg) -def cfd_healthcheck(discr, eos, state, step=0, t=0, freq=-1): +def sim_healthcheck(discr, eos, state, step=0, t=0, freq=-1): """Check the global health of the fluids state. Determine the health of a state by inspecting for unphyiscal diff --git a/test/test_callbacks.py b/test/test_callbacks.py index 49db3eb63..ffbb43182 100644 --- a/test/test_callbacks.py +++ b/test/test_callbacks.py @@ -40,7 +40,7 @@ def test_basic_cfd_healthcheck(actx_factory): - from mirgecom.simutil import cfd_healthcheck + from mirgecom.simutil import sim_healthcheck actx = actx_factory() nel_1d = 4 @@ -70,7 +70,7 @@ def test_basic_cfd_healthcheck(actx_factory): from mirgecom.exceptions import SimulationHealthError with pytest.raises(SimulationHealthError): - cfd_healthcheck(discr, eos, q, freq=1) + sim_healthcheck(discr, eos, q, freq=1) # Let's make another very bad state (nans) mass = 1*ones @@ -81,7 +81,7 @@ def test_basic_cfd_healthcheck(actx_factory): q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) with pytest.raises(SimulationHealthError): - cfd_healthcheck(discr, eos, q, freq=1) + sim_healthcheck(discr, eos, q, freq=1) # Let's make one last very bad state (inf) mass = 1*ones @@ -92,7 +92,7 @@ def test_basic_cfd_healthcheck(actx_factory): q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) with pytest.raises(SimulationHealthError): - cfd_healthcheck(discr, eos, q, freq=1) + sim_healthcheck(discr, eos, q, freq=1) def test_analytic_comparison(actx_factory): From 8284d8aadb2cd6685618649f56513c0a6a5adc36 Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Mon, 7 Jun 2021 18:05:33 -0500 Subject: [PATCH 127/385] Simplify drivers; move exception handling into sim_checkpoint --- examples/autoignition-mpi.py | 34 +--- examples/lump-mpi.py | 36 +---- examples/mixture-mpi.py | 36 +---- examples/pulse-mpi.py | 36 +---- examples/scalar-lump-mpi.py | 36 +---- examples/sod-mpi.py | 36 +---- examples/vortex-mpi.py | 36 +---- mirgecom/simutil.py | 304 ++++++++++++++++++++--------------- 8 files changed, 201 insertions(+), 353 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 6c65d28ba..2a6a00ffe 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -39,13 +39,10 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, - sim_visualization, sim_checkpoint, - sim_healthcheck, check_step, generate_and_distribute_mesh ) -from mirgecom.exceptions import SynchronizedError from mirgecom.fluid import split_conserved from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -85,7 +82,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 1 nviz = 5 - ncheck = 1 rank = 0 checkpoint_t = current_t current_step = 0 @@ -236,30 +232,12 @@ def my_checkpoint(step, t, dt, state): cv = split_conserved(dim, state) reaction_rates = eos.get_production_rates(cv) viz_fields = [("reaction_rates", reaction_rates)] - try: - # Check the health of the simulation - sim_healthcheck(discr, eos, state, - step=step, t=t, freq=ncheck) - # Perform checkpointing - sim_checkpoint(discr, eos, state, - step=step, t=t, dt=dt, freq=nstatus, - constant_cfl=constant_cfl) - # Visualize - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=nviz, - viz_fields=viz_fields) - except SynchronizedError as err: - # Log crash error message - if rank == 0: - logger.info(str(err)) - logger.info("Visualizing crashed state ...") - # Write out crashed field - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=1, - viz_fields=viz_fields) - raise err + # Perform status checkpointing + sim_checkpoint(discr, visualizer, eos, q=state, + vizname=casename, step=step, + t=t, dt=dt, nstatus=nstatus, nviz=nviz, + constant_cfl=constant_cfl, + viz_fields=viz_fields) return state current_step, current_t, current_state = \ diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 2e1e9cb88..54229eaab 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -39,13 +39,9 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, - sim_visualization, sim_checkpoint, - sim_healthcheck, - compare_with_analytic_solution, generate_and_distribute_mesh ) -from mirgecom.exceptions import SynchronizedError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -85,7 +81,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 1 nviz = 1 - ncheck = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -135,33 +130,10 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - try: - # Check the health of the simulation - sim_healthcheck(discr, eos, state, - step=step, t=t, freq=ncheck) - # Perform checkpointing - sim_checkpoint(discr, eos, state, - step=step, t=t, dt=dt, freq=nstatus, - constant_cfl=constant_cfl) - # Compare with analytic result - compare_with_analytic_solution(discr, eos, state, - exact_soln=initializer, - step=step, t=t, freq=nstatus, - exittol=exittol) - # Visualize - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=nviz) - except SynchronizedError as err: - # Log crash error message - if rank == 0: - logger.info(str(err)) - logger.info("Visualizing crashed state ...") - # Write out crashed field - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=1) - raise err + sim_checkpoint(discr, visualizer, eos, q=state, + exact_soln=initializer, vizname=casename, step=step, + t=t, dt=dt, nstatus=nstatus, nviz=nviz, + exittol=exittol, constant_cfl=constant_cfl) return state current_step, current_t, current_state = \ diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 871897c53..28ee4033b 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -39,13 +39,9 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, - sim_visualization, sim_checkpoint, - sim_healthcheck, - compare_with_analytic_solution, generate_and_distribute_mesh ) -from mirgecom.exceptions import SynchronizedError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -82,7 +78,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 1 nviz = 1 - ncheck = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -155,33 +150,10 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - try: - # Check the health of the simulation - sim_healthcheck(discr, eos, state, - step=step, t=t, freq=ncheck) - # Perform checkpointing - sim_checkpoint(discr, eos, state, - step=step, t=t, dt=dt, freq=nstatus, - constant_cfl=constant_cfl) - # Compare with analytic result - compare_with_analytic_solution(discr, eos, state, - exact_soln=initializer, - step=step, t=t, freq=nstatus, - exittol=exittol) - # Visualize - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=nviz) - except SynchronizedError as err: - # Log crash error message - if rank == 0: - logger.info(str(err)) - logger.info("Visualizing crashed state ...") - # Write out crashed field - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=1) - raise err + sim_checkpoint(discr, visualizer, eos, q=state, + exact_soln=initializer, vizname=casename, step=step, + t=t, dt=dt, nstatus=nstatus, nviz=nviz, + exittol=exittol, constant_cfl=constant_cfl) return state current_step, current_t, current_state = \ diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 54012c178..1335a092e 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -40,13 +40,9 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, - sim_visualization, sim_checkpoint, - sim_healthcheck, - compare_with_analytic_solution, generate_and_distribute_mesh ) -from mirgecom.exceptions import SynchronizedError from mirgecom.io import make_init_message from mirgecom.integrators import rk4_step @@ -93,7 +89,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 10 nviz = 10 - ncheck = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -157,33 +152,10 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - try: - # Check the health of the simulation - sim_healthcheck(discr, eos, state, - step=step, t=t, freq=ncheck) - # Perform checkpointing - sim_checkpoint(discr, eos, state, - step=step, t=t, dt=dt, freq=nstatus, - constant_cfl=constant_cfl) - # Compare with analytic result - compare_with_analytic_solution(discr, eos, state, - exact_soln=initializer, - step=step, t=t, freq=nstatus, - exittol=exittol) - # Visualize - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=nviz) - except SynchronizedError as err: - # Log crash error message - if rank == 0: - logger.info(str(err)) - logger.info("Visualizing crashed state ...") - # Write out crashed field - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=1) - raise err + sim_checkpoint(discr, visualizer, eos, q=state, + exact_soln=initializer, vizname=casename, step=step, + t=t, dt=dt, nstatus=nstatus, nviz=nviz, + exittol=exittol, constant_cfl=constant_cfl) return state current_step, current_t, current_state = \ diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index be0586937..38aab0011 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -40,13 +40,9 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, - sim_visualization, sim_checkpoint, - sim_healthcheck, - compare_with_analytic_solution, generate_and_distribute_mesh ) -from mirgecom.exceptions import SynchronizedError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -79,7 +75,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 1 nviz = 1 - ncheck = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -145,33 +140,10 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - try: - # Check the health of the simulation - sim_healthcheck(discr, eos, state, - step=step, t=t, freq=ncheck) - # Perform checkpointing - sim_checkpoint(discr, eos, state, - step=step, t=t, dt=dt, freq=nstatus, - constant_cfl=constant_cfl) - # Compare with analytic result - compare_with_analytic_solution(discr, eos, state, - exact_soln=initializer, - step=step, t=t, freq=nstatus, - exittol=exittol) - # Visualize - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=nviz) - except SynchronizedError as err: - # Log crash error message - if rank == 0: - logger.info(str(err)) - logger.info("Visualizing crashed state ...") - # Write out crashed field - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=1) - raise err + sim_checkpoint(discr, visualizer, eos, q=state, + exact_soln=initializer, vizname=casename, step=step, + t=t, dt=dt, nstatus=nstatus, nviz=nviz, + exittol=exittol, constant_cfl=constant_cfl) return state current_step, current_t, current_state = \ diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 59d2415e4..deee0677f 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -38,13 +38,9 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, - sim_visualization, sim_checkpoint, - sim_healthcheck, - compare_with_analytic_solution, generate_and_distribute_mesh ) -from mirgecom.exceptions import SynchronizedError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -82,7 +78,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 10 nviz = 10 - ncheck = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -134,33 +129,10 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - try: - # Check the health of the simulation - sim_healthcheck(discr, eos, state, - step=step, t=t, freq=ncheck) - # Perform checkpointing - sim_checkpoint(discr, eos, state, - step=step, t=t, dt=dt, freq=nstatus, - constant_cfl=constant_cfl) - # Compare with analytic result - compare_with_analytic_solution(discr, eos, state, - exact_soln=initializer, - step=step, t=t, freq=nstatus, - exittol=exittol) - # Visualize - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=nviz) - except SynchronizedError as err: - # Log crash error message - if rank == 0: - logger.info(str(err)) - logger.info("Visualizing crashed state ...") - # Write out crashed field - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=1) - raise err + sim_checkpoint(discr, visualizer, eos, q=state, + exact_soln=initializer, vizname=casename, step=step, + t=t, dt=dt, nstatus=nstatus, nviz=nviz, + exittol=exittol, constant_cfl=constant_cfl) return state current_step, current_t, current_state = \ diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 9fbe01a7a..0a6de787a 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -40,13 +40,9 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, - sim_visualization, sim_checkpoint, - sim_healthcheck, - compare_with_analytic_solution, generate_and_distribute_mesh ) -from mirgecom.exceptions import SynchronizedError from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -107,7 +103,6 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal constant_cfl = False nstatus = 10 nviz = 10 - ncheck = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -181,33 +176,10 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - try: - # Check the health of the simulation - sim_healthcheck(discr, eos, state, - step=step, t=t, freq=ncheck) - # Perform checkpointing - sim_checkpoint(discr, eos, state, - step=step, t=t, dt=dt, freq=nstatus, - constant_cfl=constant_cfl) - # Compare with analytic result - compare_with_analytic_solution(discr, eos, state, - exact_soln=initializer, - step=step, t=t, freq=nstatus, - exittol=exittol) - # Visualize - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=nviz) - except SynchronizedError as err: - # Log crash error message - if rank == 0: - logger.info(str(err)) - logger.info("Visualizing crashed state ...") - # Write out crashed field - sim_visualization(discr, eos, state, - visualizer, vizname=casename, - step=step, t=t, freq=1) - raise err + sim_checkpoint(discr, visualizer, eos, q=state, + exact_soln=initializer, vizname=casename, step=step, + t=t, dt=dt, nstatus=nstatus, nviz=nviz, + exittol=exittol, constant_cfl=constant_cfl) return state current_step, current_t, current_state = \ diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 7e5d52dbb..d88fcc9cd 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -50,7 +50,7 @@ from meshmode.dof_array import thaw from mirgecom.io import make_status_message from mirgecom.inviscid import get_inviscid_timestep # bad smell? -from mirgecom.exceptions import SimulationHealthError +from mirgecom.exceptions import SynchronizedError, SimulationHealthError logger = logging.getLogger(__name__) @@ -89,7 +89,7 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, def sim_visualization(discr, eos, state, visualizer, vizname, - step=0, t=0, freq=-1, + step=0, t=0, exact_soln=None, viz_fields=None, overwrite=False, vis_timer=None): """Visualize the simulation fields. @@ -111,100 +111,146 @@ def sim_visualization(discr, eos, state, visualizer, vizname, visualizer: A :class:`meshmode.discretization.visualization.Visualizer` VTK output object. - freq: int - An integer denoting the step frequency. """ - if check_step(step=step, interval=freq): + from contextlib import nullcontext + from mirgecom.fluid import split_conserved + from mirgecom.io import make_rank_fname, make_par_fname + + cv = split_conserved(discr.dim, state) + dependent_vars = eos.dependent_vars(cv) + + io_fields = [ + ("cv", cv), + ("dv", dependent_vars) + ] + if exact_soln is not None: + actx = cv.mass.array_context + nodes = thaw(actx, discr.nodes()) + expected_state = exact_soln(x_vec=nodes, t=t, eos=eos) + exact_list = [ + ("exact_soln", expected_state), + ] + io_fields.extend(exact_list) - from contextlib import nullcontext - from mirgecom.fluid import split_conserved - from mirgecom.io import make_rank_fname, make_par_fname + if viz_fields is not None: + io_fields.extend(viz_fields) - cv = split_conserved(discr.dim, state) - dependent_vars = eos.dependent_vars(cv) + comm = discr.mpi_communicator + rank = 0 + if comm is not None: + rank = comm.Get_rank() - io_fields = [ - ("cv", cv), - ("dv", dependent_vars) - ] - if exact_soln is not None: - actx = cv.mass.array_context - nodes = thaw(actx, discr.nodes()) - expected_state = exact_soln(x_vec=nodes, t=t, eos=eos) - exact_list = [ - ("exact_soln", expected_state), - ] - io_fields.extend(exact_list) - - if viz_fields is not None: - io_fields.extend(viz_fields) - - comm = discr.mpi_communicator - rank = 0 - if comm is not None: - rank = comm.Get_rank() - - rank_fn = make_rank_fname(basename=vizname, rank=rank, step=step, t=t) - - if vis_timer: - ctm = vis_timer.start_sub_timer() - else: - ctm = nullcontext() - - with ctm: - visualizer.write_parallel_vtk_file( - comm, rank_fn, io_fields, - overwrite=overwrite, - par_manifest_filename=make_par_fname( - basename=vizname, step=step, t=t - ) + rank_fn = make_rank_fname(basename=vizname, rank=rank, step=step, t=t) + + if vis_timer: + ctm = vis_timer.start_sub_timer() + else: + ctm = nullcontext() + + with ctm: + visualizer.write_parallel_vtk_file( + comm, rank_fn, io_fields, + overwrite=overwrite, + par_manifest_filename=make_par_fname( + basename=vizname, step=step, t=t ) + ) -def sim_checkpoint(discr, eos, state, step=0, t=0, dt=0, - cfl=1.0, freq=-1, constant_cfl=False): +def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, + step=0, t=0, dt=0, cfl=1.0, nstatus=-1, nviz=-1, exittol=1e-16, + constant_cfl=False, viz_fields=None, overwrite=False, + vis_timer=None): """Checkpoint the simulation status. Checkpoints the simulation status by reporting relevant diagnostic - quantities, such as pressure/temperature. + quantities, such as pressure/temperature, and visualization. Parameters ---------- eos: mirgecom.eos.GasEOS Implementing the pressure and temperature functions for - returning pressure and temperature as a function of the state. - state + returning pressure and temperature as a function of the state *q*. + q State array which expects at least the conserved quantities (mass, energy, momentum) for the fluid at each point. For multi-component fluids, the conserved quantities should include (mass, energy, momentum, species_mass), where *species_mass* is a vector of species masses. - freq: int - An integer denoting the step frequency. + freq: nstatus + An integer denoting the step frequency for performing status checks. + freq: nviz + An integer denoting the step frequency for writing vtk output. """ - if check_step(step=step, interval=freq): + exception = None + comm = discr.mpi_communicator + rank = 0 + if comm is not None: + rank = comm.Get_rank() + + try: + # Status checks + if check_step(step=step, interval=nstatus): + from mirgecom.fluid import split_conserved + + # if constant_cfl is False: + # current_cfl = get_inviscid_cfl(discr=discr, q=q, + # eos=eos, dt=dt) + + cv = split_conserved(discr.dim, q) + dependent_vars = eos.dependent_vars(cv) + msg = make_status_message(discr=discr, + t=t, step=step, dt=dt, cfl=cfl, + dependent_vars=dependent_vars) + if rank == 0: + logger.info(msg) + + # Check the health of the simulation + sim_healthcheck(discr, eos, q, step=step, t=t) + + # Compare with exact solution, if provided + if exact_soln is not None: + compare_with_analytic_solution( + discr, eos, q, exact_soln, + step=step, t=t, exittol=exittol + ) + + # Visualization + if check_step(step=step, interval=nviz): + sim_visualization( + discr, eos, q, visualizer, vizname, + step=step, t=t, + exact_soln=exact_soln, viz_fields=viz_fields, + overwrite=overwrite, vis_timer=vis_timer + ) + except SynchronizedError as err: + exception = err + + terminate = True if exception is not None else False - from mirgecom.fluid import split_conserved + if comm is None: + if terminate: + raise exception + return - # if constant_cfl is False: - # current_cfl = get_inviscid_cfl(discr=discr, q=state, - # eos=eos, dt=dt) + from mpi4py import MPI - comm = discr.mpi_communicator - rank = 0 - if comm is not None: - rank = comm.Get_rank() + terminate = comm.allreduce(terminate, MPI.LOR) - cv = split_conserved(discr.dim, state) - dependent_vars = eos.dependent_vars(cv) - msg = make_status_message(discr=discr, - t=t, step=step, dt=dt, cfl=cfl, - dependent_vars=dependent_vars) + if terminate: + # Log crash error message if rank == 0: - logger.info(msg) + logger.info(str(exception)) + logger.info("Visualizing crashed state ...") + # Write out crashed field + sim_visualization(discr, eos, q, + visualizer, vizname=vizname, + step=step, t=t, + viz_fields=viz_fields) + raise exception -def sim_healthcheck(discr, eos, state, step=0, t=0, freq=-1): +def sim_healthcheck(discr, eos, state, step=0, t=0): """Check the global health of the fluids state. Determine the health of a state by inspecting for unphyiscal @@ -221,48 +267,44 @@ def sim_healthcheck(discr, eos, state, step=0, t=0, freq=-1): fluids, the conserved quantities should include (mass, energy, momentum, species_mass), where *species_mass* is a vector of species masses. - freq: int - An integer denoting the step frequency. """ - if check_step(step=step, interval=freq): - - from mirgecom.fluid import split_conserved - import grudge.op as op - - # NOTE: Derived quantities are functions of the conserved variables. - # Therefore is it sufficient to check for unphysical values of - # temperature and pressure. - cv = split_conserved(discr.dim, state) - dependent_vars = eos.dependent_vars(cv) - pressure = dependent_vars.pressure - temperature = dependent_vars.temperature - - # Check for NaN - if (np.isnan(op.nodal_sum_loc(discr, "vol", pressure)) - or np.isnan(op.nodal_sum_loc(discr, "vol", temperature))): - raise SimulationHealthError( - message=("Simulation exited abnormally; detected a NaN.") - ) + from mirgecom.fluid import split_conserved + import grudge.op as op + + # NOTE: Derived quantities are functions of the conserved variables. + # Therefore is it sufficient to check for unphysical values of + # temperature and pressure. + cv = split_conserved(discr.dim, state) + dependent_vars = eos.dependent_vars(cv) + pressure = dependent_vars.pressure + temperature = dependent_vars.temperature + + # Check for NaN + if (np.isnan(op.nodal_sum_loc(discr, "vol", pressure)) + or np.isnan(op.nodal_sum_loc(discr, "vol", temperature))): + raise SimulationHealthError( + message=("Simulation exited abnormally; detected a NaN.") + ) - # Check for non-positivity - if (op.nodal_min_loc(discr, "vol", pressure) < 0 - or op.nodal_min_loc(discr, "vol", temperature) < 0): - raise SimulationHealthError( - message=("Simulation exited abnormally; " - "found non-positive values for pressure or temperature.") - ) + # Check for non-positivity + if (op.nodal_min_loc(discr, "vol", pressure) < 0 + or op.nodal_min_loc(discr, "vol", temperature) < 0): + raise SimulationHealthError( + message=("Simulation exited abnormally; " + "found non-positive values for pressure or temperature.") + ) - # Check for blow-up - if (op.nodal_sum_loc(discr, "vol", pressure) == np.inf - or op.nodal_sum_loc(discr, "vol", temperature) == np.inf): - raise SimulationHealthError( - message=("Simulation exited abnormally; " - "derived quantities are not finite.") - ) + # Check for blow-up + if (op.nodal_sum_loc(discr, "vol", pressure) == np.inf + or op.nodal_sum_loc(discr, "vol", temperature) == np.inf): + raise SimulationHealthError( + message=("Simulation exited abnormally; " + "derived quantities are not finite.") + ) def compare_with_analytic_solution(discr, eos, state, exact_soln, - step=0, t=0, freq=-1, exittol=None): + step=0, t=0, exittol=None): """Compute the infinite norm of the problem residual. Computes the infinite norm of the residual with respect to a specified @@ -284,40 +326,36 @@ def compare_with_analytic_solution(discr, eos, state, exact_soln, A callable for the exact solution with signature: ``exact_soln(x_vec, t, eos)`` where `x_vec` are the nodes, `t` is time, and `eos` is a :class:`mirgecom.eos.GasEOS`. - freq: int - An integer denoting the step frequency. """ - if check_step(step=step, interval=freq): + if exittol is None: + exittol = 1e-16 + + actx = discr._setup_actx + nodes = thaw(actx, discr.nodes()) + expected_state = exact_soln(x_vec=nodes, t=t, eos=eos) + exp_resid = state - expected_state + norms = [discr.norm(v, np.inf) for v in exp_resid] + + comm = discr.mpi_communicator + rank = 0 + if comm is not None: + rank = comm.Get_rank() + + statusmesg = ( + f"Errors: {step=} {t=}\n" + f"------- errors = " + + ", ".join("%.3g" % err_norm for err_norm in norms) + ) - if exittol is None: - exittol = 1e-16 + if rank == 0: + logger.info(statusmesg) - actx = discr._setup_actx - nodes = thaw(actx, discr.nodes()) - expected_state = exact_soln(x_vec=nodes, t=t, eos=eos) - exp_resid = state - expected_state - norms = [discr.norm(v, np.inf) for v in exp_resid] - - comm = discr.mpi_communicator - rank = 0 - if comm is not None: - rank = comm.Get_rank() - - statusmesg = ( - f"Errors: {step=} {t=}\n" - f"------- errors = " - + ", ".join("%.3g" % err_norm for err_norm in norms) + if max(norms) > exittol: + raise SimulationHealthError( + message=("Simulation exited abnormally; " + "solution doesn't agree with analytic result.") ) - if rank == 0: - logger.info(statusmesg) - - if max(norms) > exittol: - raise SimulationHealthError( - message=("Simulation exited abnormally; " - "solution doesn't agree with analytic result.") - ) - def generate_and_distribute_mesh(comm, generate_mesh): """Generate a mesh and distribute it among all ranks in *comm*. From cba127b9a2dd999394099bfab138d692637f9eba Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Mon, 7 Jun 2021 19:18:30 -0500 Subject: [PATCH 128/385] Fix callback tests --- test/test_callbacks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_callbacks.py b/test/test_callbacks.py index ffbb43182..7ed1b9663 100644 --- a/test/test_callbacks.py +++ b/test/test_callbacks.py @@ -70,7 +70,7 @@ def test_basic_cfd_healthcheck(actx_factory): from mirgecom.exceptions import SimulationHealthError with pytest.raises(SimulationHealthError): - sim_healthcheck(discr, eos, q, freq=1) + sim_healthcheck(discr, eos, q) # Let's make another very bad state (nans) mass = 1*ones @@ -81,7 +81,7 @@ def test_basic_cfd_healthcheck(actx_factory): q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) with pytest.raises(SimulationHealthError): - sim_healthcheck(discr, eos, q, freq=1) + sim_healthcheck(discr, eos, q) # Let's make one last very bad state (inf) mass = 1*ones @@ -92,7 +92,7 @@ def test_basic_cfd_healthcheck(actx_factory): q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) with pytest.raises(SimulationHealthError): - sim_healthcheck(discr, eos, q, freq=1) + sim_healthcheck(discr, eos, q) def test_analytic_comparison(actx_factory): @@ -125,4 +125,4 @@ def test_analytic_comparison(actx_factory): from mirgecom.exceptions import SimulationHealthError with pytest.raises(SimulationHealthError): - compare_with_analytic_solution(discr, eos, q, Vortex2D(), freq=1) + compare_with_analytic_solution(discr, eos, q, Vortex2D()) From b58594c1b18f501faa687f7b5a31014670b95a10 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Mon, 7 Jun 2021 23:03:48 -0500 Subject: [PATCH 129/385] Use better dt_utils function. --- mirgecom/inviscid.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index d004a4eb8..74c4510b3 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -98,17 +98,12 @@ def get_inviscid_timestep(discr, eos, q): class:`~meshmode.dof_array.DOFArray` The maximum stable timestep at each node. """ - from grudge.dt_utils import (dt_non_geometric_factor, - dt_geometric_factors) + from grudge.dt_utils import estimate_local_timestep from mirgecom.fluid import compute_wavespeed dim = discr.dim cv = split_conserved(dim, q) - - return ( - dt_non_geometric_factor(discr) * dt_geometric_factors(discr) - / compute_wavespeed(dim, eos, cv) - ) + return estimate_local_timestep(discr, compute_wavespeed(dim, eos, cv)) def get_inviscid_cfl(discr, eos, dt, q): From e2dc67b2a1dbbf61096b1e4c643244f040804c7f Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 8 Jun 2021 00:03:50 -0500 Subject: [PATCH 130/385] Use grudge@nongeo-factors-per-group util inducer/grudge/#121 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5a5d4b277..8a73fd734 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ psutil --editable git+https://github.com/inducer/modepy.git#egg=modepy --editable git+https://github.com/inducer/arraycontext.git#egg=arraycontext --editable git+https://github.com/inducer/meshmode.git#egg=meshmode ---editable git+https://github.com/inducer/grudge.git#egg=grudge +--editable git+https://github.com/inducer/grudge.git@nongeo-factors-per-group#egg=grudge --editable git+https://github.com/inducer/pytato.git#egg=pytato --editable git+https://github.com/ecisneros8/pyrometheus.git#egg=pyrometheus --editable git+https://github.com/illinois-ceesd/logpyle.git#egg=logpyle From 4c938fa2cb2eef71d83aff54792fc14ad20b86f5 Mon Sep 17 00:00:00 2001 From: Thomas Gibson Date: Tue, 8 Jun 2021 13:45:27 -0500 Subject: [PATCH 131/385] Clean up sim_checkpoint function --- mirgecom/simutil.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index d88fcc9cd..c5222576a 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -177,9 +177,9 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, fluids, the conserved quantities should include (mass, energy, momentum, species_mass), where *species_mass* is a vector of species masses. - freq: nstatus + nstatus: int An integer denoting the step frequency for performing status checks. - freq: nviz + nviz: int An integer denoting the step frequency for writing vtk output. """ exception = None @@ -228,20 +228,15 @@ def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, terminate = True if exception is not None else False - if comm is None: - if terminate: - raise exception - return - - from mpi4py import MPI + if comm is not None: + from mpi4py import MPI - terminate = comm.allreduce(terminate, MPI.LOR) + terminate = comm.allreduce(terminate, MPI.LOR) if terminate: - # Log crash error message if rank == 0: - logger.info(str(exception)) logger.info("Visualizing crashed state ...") + # Write out crashed field sim_visualization(discr, eos, q, visualizer, vizname=vizname, From ece824f53c85a26328673042b41f7427dd1c2b85 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 9 Jun 2021 06:03:55 -0500 Subject: [PATCH 132/385] Initial stab at using CV array container inside AV on NS. --- examples/doublemach-mpi.py | 15 +++++++------ mirgecom/artificial_viscosity.py | 10 ++++----- mirgecom/boundary.py | 27 +++++++++++++++++++++++ mirgecom/initializers.py | 2 +- test/test_av.py | 38 +++++++++++++++++++++++++------- 5 files changed, 70 insertions(+), 22 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 901bbadd1..7a9885b75 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -38,7 +38,6 @@ from mirgecom.navierstokes import ns_operator -from mirgecom.fluid import split_conserved from mirgecom.artificial_viscosity import ( av_operator, smoothness_indicator @@ -54,7 +53,10 @@ from mirgecom.integrators import rk4_step from mirgecom.steppers import advance_state -from mirgecom.boundary import AdiabaticSlipBoundary, PrescribedBoundary +from mirgecom.boundary import ( + AdiabaticNoslipMovingBoundary, + PrescribedBoundary +) from mirgecom.initializers import DoubleMachReflection from mirgecom.eos import IdealSingleGas from mirgecom.transport import SimpleTransport @@ -138,8 +140,8 @@ def main(ctx_factory=cl.create_some_context): DTAG_BOUNDARY("ic1"): PrescribedBoundary(initializer), DTAG_BOUNDARY("ic2"): PrescribedBoundary(initializer), DTAG_BOUNDARY("ic3"): PrescribedBoundary(initializer), - DTAG_BOUNDARY("wall"): AdiabaticSlipBoundary(), - DTAG_BOUNDARY("out"): AdiabaticSlipBoundary(), + DTAG_BOUNDARY("wall"): AdiabaticNoslipMovingBoundary(), + DTAG_BOUNDARY("out"): AdiabaticNoslipMovingBoundary(), } constant_cfl = False nstatus = 10 @@ -212,14 +214,13 @@ def my_rhs(t, state): ) def my_checkpoint(step, t, dt, state): - cv = split_conserved(dim, state) - tagged_cells = smoothness_indicator(discr, cv.mass, s0=s0, kappa=kappa) + tagged_cells = smoothness_indicator(discr, state.mass, s0=s0, kappa=kappa) viz_fields = [("tagged cells", tagged_cells)] return sim_checkpoint( discr, visualizer, eos, - q=state, + cv=state, vizname=casename, step=step, t=t, diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 3c13a4e6e..3fa687766 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -209,12 +209,10 @@ def av_operator(discr, boundaries, q, alpha, boundary_kwargs=None, **kwargs): q_bnd_flux = (_facial_flux_q(discr, q_tpair=interior_trace_pair(discr, q)) + sum(_facial_flux_q(discr, q_tpair=pb_tpair) for pb_tpair in cross_rank_trace_pairs(discr, q))) - q_bnd_flux2 = sum(bnd.q_boundary_flux(discr, btag, q=q, **boundary_kwargs) + q_bnd_flux2 = sum(bnd.soln_gradient_flux(discr, btag, soln=q, **boundary_kwargs) for btag, bnd in boundaries.items()) - if isinstance(q, np.ndarray): - q_bnd_flux2 = np.stack(q_bnd_flux2) - if isinstance(q_bnd_flux2, ConservedVars): - q_bnd_flux2 = q_bnd_flux2.join() + # if isinstance(q, np.ndarray): + # q_bnd_flux2 = np.stack(q_bnd_flux2) q_bnd_flux = q_bnd_flux + q_bnd_flux2 # Compute R @@ -228,7 +226,7 @@ def av_operator(discr, boundaries, q, alpha, boundary_kwargs=None, **kwargs): r_bnd_flux = (_facial_flux_r(discr, r_tpair=interior_trace_pair(discr, r)) + sum(_facial_flux_r(discr, r_tpair=pb_tpair) for pb_tpair in cross_rank_trace_pairs(discr, r)) - + sum(bnd.s_boundary_flux(discr, btag, grad_q=r, **boundary_kwargs) + + sum(bnd.av_flux(discr, btag, diffusion=r, **boundary_kwargs) for btag, bnd in boundaries.items())) # Return the AV RHS term diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 912e05ae1..522bb8daa 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -129,6 +129,8 @@ class PrescribedInviscidBoundary(FluidBC): .. automethod:: __init__ .. automethod:: boundary_pair .. automethod:: inviscid_boundary_flux + .. automethod:: soln_gradient_flux + .. automethod:: av_flux """ def __init__(self, inviscid_boundary_flux_func=None, boundary_pair_func=None, @@ -207,6 +209,11 @@ def q_boundary_flux(self, discr, btag, cv, **kwargs): return self._boundary_quantity(discr, btag=btag, quantity=flux_weak, **kwargs) + def soln_gradient_flux(self, discr, btag, soln, **kwargs): + """Get the flux for solution gradient with AV API.""" + cv = make_conserved(discr.dim, q=soln) + return self.q_boundary_flux(discr, btag, cv, **kwargs).join() + def s_boundary_flux(self, discr, btag, grad_cv, **kwargs): r"""Get $\nabla\mathbf{Q}$ flux across the boundary faces.""" actx = grad_cv.array_context @@ -227,6 +234,11 @@ def s_boundary_flux(self, discr, btag, grad_cv, **kwargs): **kwargs ) + def av_flux(self, discr, btag, diffusion, **kwargs): + """Get the diffusive fluxes for the AV operator API.""" + diff_cv = make_conserved(discr.dim, q=diffusion) + return self.s_boundary_flux(discr, btag, diff_cv, **kwargs).join() + def t_boundary_flux(self, discr, btag, cv, eos, **kwargs): """Get the "temperature flux" through boundary *btag*.""" cv_minus = discr.project("vol", btag, cv) @@ -379,6 +391,21 @@ def adiabatic_slip_pair(self, discr, cv, btag, **kwargs): momentum=ext_mom, species_mass=int_cv.species_mass) return TracePair(btag, interior=int_cv, exterior=ext_cv) + def exterior_grad_q(self, nodes, nhat, grad_cv, **kwargs): + """Get the exterior grad(Q) on the boundary.""" + # Grab some boundary-relevant data + num_equations, dim = grad_cv.mass.shape + + # Subtract 2*wall-normal component of q + # to enforce q=0 on the wall + s_mom_normcomp = np.outer(nhat, np.dot(grad_cv.momentum, nhat)) + s_mom_flux = grad_cv.momentum - 2*s_mom_normcomp + + # flip components to set a neumann condition + return make_conserved(dim, mass=-grad_cv.mass, energy=-grad_cv.energy, + momentum=-s_mom_flux, + species_mass=-grad_cv.species_mass) + class AdiabaticNoslipMovingBoundary(PrescribedInviscidBoundary): r"""Boundary condition implementing a noslip moving boundary. diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index f3b317256..048958dcb 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -382,7 +382,7 @@ def __call__(self, x_vec, *, eos=None, time=0, **kwargs): mom = mass * vel energy = rhoe + .5*mass*np.dot(vel, vel) - return join_conserved(dim=2, mass=mass, energy=energy, momentum=mom) + return make_conserved(dim=2, mass=mass, energy=energy, momentum=mom) class Lump: diff --git a/test/test_av.py b/test/test_av.py index 0390bc960..626893753 100644 --- a/test/test_av.py +++ b/test/test_av.py @@ -36,7 +36,6 @@ av_operator, smoothness_indicator ) -from mirgecom.boundary import DummyBoundary from grudge.eager import EagerDGDiscretization from pyopencl.tools import ( # noqa pytest_generate_tests_for_pyopencl as pytest_generate_tests, @@ -176,22 +175,45 @@ def test_artificial_viscosity(ctx_factory, dim, order): nodes = thaw(actx, discr.nodes()) zeros = discr.zeros(actx) - boundaries = {BTAG_ALL: DummyBoundary()} + class TestBoundary: + def soln_gradient_flux(self, disc, btag, soln, **kwargs): + soln_int = disc.project("vol", btag, soln) + from grudge.trace_pair import TracePair + bnd_pair = TracePair(btag, + interior=soln_int, + exterior=soln_int) + nhat = thaw(actx, disc.normal(btag)) + from mirgecom.flux import central_scalar_flux + flux_weak = central_scalar_flux(bnd_pair, normal=nhat) + return disc.project(btag, "all_faces", flux_weak) + + def av_flux(self, disc, btag, diffusion, **kwargs): + nhat = thaw(actx, disc.normal(btag)) + grad_soln_minus = discr.project("vol", btag, diffusion) + grad_soln_plus = grad_soln_minus + from grudge.trace_pair import TracePair + bnd_grad_pair = TracePair(btag, interior=grad_soln_minus, + exterior=grad_soln_plus) + from mirgecom.flux import central_vector_flux + flux_weak = central_vector_flux(bnd_grad_pair, normal=nhat) + return disc.project(btag, "all_faces", flux_weak) + + boundaries = {BTAG_ALL: TestBoundary()} # Uniform field return 0 rhs - q = zeros + 1.0 - rhs = av_operator(discr, boundaries=boundaries, q=q, alpha=1.0, s0=-np.inf) + soln = zeros + 1.0 + rhs = av_operator(discr, boundaries=boundaries, q=soln, alpha=1.0, s0=-np.inf) err = discr.norm(rhs, np.inf) assert err < tolerance # Linear field return 0 rhs - q = nodes[0] - rhs = av_operator(discr, boundaries=boundaries, q=q, alpha=1.0, s0=-np.inf) + soln = nodes[0] + rhs = av_operator(discr, boundaries=boundaries, q=soln, alpha=1.0, s0=-np.inf) err = discr.norm(rhs, np.inf) assert err < tolerance # Quadratic field return constant 2 - q = np.dot(nodes, nodes) - rhs = av_operator(discr, boundaries=boundaries, q=q, alpha=1.0, s0=-np.inf) + soln = np.dot(nodes, nodes) + rhs = av_operator(discr, boundaries=boundaries, q=soln, alpha=1.0, s0=-np.inf) err = discr.norm(2.*dim-rhs, np.inf) assert err < tolerance From 8f41a32b9fcacc9f88e860a39993622bd6562a37 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 9 Jun 2021 06:32:03 -0500 Subject: [PATCH 133/385] Update ns api in doublemach example --- examples/doublemach-mpi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 7a9885b75..3fac69948 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -206,9 +206,9 @@ def main(ctx_factory=cl.create_some_context): def my_rhs(t, state): return ns_operator( - discr, q=state, t=t, boundaries=boundaries, eos=eos + discr, cv=state, t=t, boundaries=boundaries, eos=eos ) + av_operator( - discr, q=state, boundaries=boundaries, + discr, q=state.join(), boundaries=boundaries, boundary_kwargs={"time": t, "eos": eos}, alpha=alpha, s0=s0, kappa=kappa ) From 964859749295e8cc73742ff461fe930048b6a15a Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 9 Jun 2021 07:03:50 -0500 Subject: [PATCH 134/385] Remove unneeded dim var --- mirgecom/inviscid.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 32d9d18bf..5059125dc 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -90,23 +90,19 @@ def inviscid_facial_flux(discr, eos, cv_tpair, local=False): "all_faces"; remaining instead on the boundary restriction. """ actx = cv_tpair.int.array_context - dim = discr.dim flux_tpair = TracePair(cv_tpair.dd, interior=inviscid_flux(discr, eos, cv_tpair.int), exterior=inviscid_flux(discr, eos, cv_tpair.ext)) lam = actx.np.maximum( - compute_wavespeed(dim, eos=eos, cv=cv_tpair.int), - compute_wavespeed(dim, eos=eos, cv=cv_tpair.ext) + compute_wavespeed(eos=eos, cv=cv_tpair.int), + compute_wavespeed(eos=eos, cv=cv_tpair.ext) ) normal = thaw(actx, discr.normal(cv_tpair.dd)) # todo: user-supplied flux routine - # flux_weak = make_conserved( - # dim, q=lfr_flux(cv_tpair, flux_tpair, normal=normal, lam=lam) - # ) flux_weak = lfr_flux(cv_tpair, flux_tpair, normal=normal, lam=lam) if local is False: From 1c81b7dcd7ccf042f332c50c5451c76ecd5c8aae Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 9 Jun 2021 07:12:42 -0500 Subject: [PATCH 135/385] Use explicit named args in call to s_bnd_flux --- mirgecom/boundary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 522bb8daa..92a2d30b4 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -237,7 +237,7 @@ def s_boundary_flux(self, discr, btag, grad_cv, **kwargs): def av_flux(self, discr, btag, diffusion, **kwargs): """Get the diffusive fluxes for the AV operator API.""" diff_cv = make_conserved(discr.dim, q=diffusion) - return self.s_boundary_flux(discr, btag, diff_cv, **kwargs).join() + return self.s_boundary_flux(discr, btag, grad_cv=diff_cv, **kwargs).join() def t_boundary_flux(self, discr, btag, cv, eos, **kwargs): """Get the "temperature flux" through boundary *btag*.""" From aa1ea7dc36c38de37741073a728eb4b79dc5c15e Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 9 Jun 2021 07:16:00 -0500 Subject: [PATCH 136/385] Correct inviscid_timestep call in vortex example --- examples/vortex-mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 4221a5b7d..76650ea3d 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -180,7 +180,7 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - local_cfl = get_inviscid_cfl(discr, eos=eos, dt=current_dt, q=state) + local_cfl = get_inviscid_cfl(discr, eos=eos, dt=current_dt, cv=state) viz_fields = [ ("cfl", local_cfl) ] From 3cff6410e752cf843eb8afceed3e68d66a1816e4 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 9 Jun 2021 08:29:10 -0500 Subject: [PATCH 137/385] Wriggle into place with the new shiny data structure --- examples/doublemach-mpi.py | 6 +++--- mirgecom/boundary.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index e46f5ebcb..e093054f8 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -50,7 +50,7 @@ ) from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point - +from mirgecom.fluid import make_conserved from mirgecom.integrators import rk4_step from mirgecom.steppers import advance_state from mirgecom.boundary import ( @@ -207,10 +207,10 @@ def main(ctx_factory=cl.create_some_context): def my_rhs(t, state): return ns_operator( discr, cv=state, t=t, boundaries=boundaries, eos=eos - ) + av_operator( + ) + make_conserved(dim, q=av_operator( discr, q=state.join(), boundaries=boundaries, boundary_kwargs={"time": t, "eos": eos}, alpha=alpha, - s0=s0, kappa=kappa + s0=s0, kappa=kappa) ) def my_checkpoint(step, t, dt, state): diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 522bb8daa..86563fd7c 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -216,7 +216,7 @@ def soln_gradient_flux(self, discr, btag, soln, **kwargs): def s_boundary_flux(self, discr, btag, grad_cv, **kwargs): r"""Get $\nabla\mathbf{Q}$ flux across the boundary faces.""" - actx = grad_cv.array_context + actx = grad_cv.mass[0].array_context boundary_discr = discr.discr_from_dd(btag) nodes = thaw(actx, boundary_discr.nodes()) nhat = thaw(actx, discr.normal(btag)) @@ -450,9 +450,9 @@ def exterior_soln(self, discr, cv, btag, **kwargs): return make_conserved(dim=dim, mass=int_cv.mass, energy=int_cv.energy, momentum=ext_mom) - def exterior_grad_q(self, nodes, nhat, grad_q, **kwargs): + def exterior_grad_q(self, nodes, nhat, grad_cv, **kwargs): """Get the exterior solution on the boundary.""" - return(-grad_q) + return(-grad_cv) class IsothermalNoSlipBoundary(PrescribedInviscidBoundary): From 68d52e70097574463cb9e52f5410ba0a6fe2e0f5 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 9 Jun 2021 10:04:33 -0500 Subject: [PATCH 138/385] Wriggle into place with new data structures. --- examples/doublemach-mpi.py | 18 ++++++++++-------- mirgecom/boundary.py | 6 +++--- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 3fac69948..2316bb03a 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -37,6 +37,7 @@ from grudge.shortcuts import make_visualizer +from mirgecom.fluid import make_conserved from mirgecom.navierstokes import ns_operator from mirgecom.artificial_viscosity import ( av_operator, @@ -207,10 +208,10 @@ def main(ctx_factory=cl.create_some_context): def my_rhs(t, state): return ns_operator( discr, cv=state, t=t, boundaries=boundaries, eos=eos - ) + av_operator( + ) + make_conserved(dim, q=av_operator( discr, q=state.join(), boundaries=boundaries, boundary_kwargs={"time": t, "eos": eos}, alpha=alpha, - s0=s0, kappa=kappa + s0=s0, kappa=kappa) ) def my_checkpoint(step, t, dt, state): @@ -251,12 +252,13 @@ def my_checkpoint(step, t, dt, state): # if current_t != checkpoint_t: if rank == 0: logger.info("Checkpointing final state ...") - my_checkpoint( - current_step, - t=current_t, - dt=(current_t - checkpoint_t), - state=current_state, - ) + + my_checkpoint( + current_step, + t=current_t, + dt=(current_t - checkpoint_t), + state=current_state, + ) if current_t - t_final < 0: raise ValueError("Simulation exited abnormally") diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 92a2d30b4..3603ce767 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -216,7 +216,7 @@ def soln_gradient_flux(self, discr, btag, soln, **kwargs): def s_boundary_flux(self, discr, btag, grad_cv, **kwargs): r"""Get $\nabla\mathbf{Q}$ flux across the boundary faces.""" - actx = grad_cv.array_context + actx = grad_cv.mass[0].array_context boundary_discr = discr.discr_from_dd(btag) nodes = thaw(actx, boundary_discr.nodes()) nhat = thaw(actx, discr.normal(btag)) @@ -450,9 +450,9 @@ def exterior_soln(self, discr, cv, btag, **kwargs): return make_conserved(dim=dim, mass=int_cv.mass, energy=int_cv.energy, momentum=ext_mom) - def exterior_grad_q(self, nodes, nhat, grad_q, **kwargs): + def exterior_grad_q(self, nodes, nhat, grad_cv, **kwargs): """Get the exterior solution on the boundary.""" - return(-grad_q) + return(-grad_cv) class IsothermalNoSlipBoundary(PrescribedInviscidBoundary): From bb607c715ca730e6994a2945e2a1499396d7fbe4 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 9 Jun 2021 10:29:42 -0500 Subject: [PATCH 139/385] Try to update everything for new API --- mirgecom/fluid.py | 3 +-- mirgecom/initializers.py | 12 +++--------- mirgecom/inviscid.py | 8 ++------ 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/mirgecom/fluid.py b/mirgecom/fluid.py index 537705a7f..86ddb5f79 100644 --- a/mirgecom/fluid.py +++ b/mirgecom/fluid.py @@ -426,7 +426,7 @@ def species_mass_fraction_gradient(discr, cv, grad_cv): return grad_y -def compute_wavespeed(dim, eos, cv: ConservedVars): +def compute_wavespeed(eos, cv: ConservedVars): r"""Return the wavespeed in the flow. The wavespeed is calculated as: @@ -438,6 +438,5 @@ def compute_wavespeed(dim, eos, cv: ConservedVars): where $\mathbf{v}$ is the flow velocity and c is the speed of sound in the fluid. """ actx = cv.array_context - v = cv.momentum / cv.mass return actx.np.sqrt(np.dot(v, v)) + eos.sound_speed(cv) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 048958dcb..b6220e2dc 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -227,14 +227,12 @@ def __init__( if self._xdir >= self._dim: self._xdir = self._dim - 1 - def __call__(self, x_vec, *, eos=None, time=0, **kwargs): + def __call__(self, x_vec, *, eos=None, **kwargs): """ Create the 1D Sod's shock solution at locations *x_vec*. Parameters ---------- - time: float - Current time at which the solution is desired (unused) x_vec: numpy.ndarray Nodal coordinates eos: :class:`mirgecom.eos.IdealSingleGas` @@ -843,14 +841,12 @@ def __init__( self._e = e self._dim = dim - def __call__(self, x_vec, *, eos=None, time=0, **kwargs): + def __call__(self, x_vec, *, eos=None, **kwargs): """ Create a uniform flow solution at locations *x_vec*. Parameters ---------- - time: float - Current time at which the solution is desired (unused) x_vec: numpy.ndarray Nodal coordinates eos: :class:`mirgecom.eos.IdealSingleGas` @@ -938,7 +934,7 @@ def __init__( self._temperature = temperature self._massfracs = massfractions - def __call__(self, x_vec, eos, *, time=0.0, **kwargs): + def __call__(self, x_vec, eos, **kwargs): """ Create the mixture state at locations *x_vec* (t is ignored). @@ -951,8 +947,6 @@ def __call__(self, x_vec, eos, *, time=0.0, **kwargs): these functions: `eos.get_density` `eos.get_internal_energy` - time: float - Time is ignored by this solution intitializer (unused) """ if x_vec.shape != (self._dim,): raise ValueError(f"Position vector has unexpected dimensionality," diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 0e8773903..3bfa9c1c7 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -90,23 +90,19 @@ def inviscid_facial_flux(discr, eos, cv_tpair, local=False): "all_faces"; remaining instead on the boundary restriction. """ actx = cv_tpair.int.array_context - dim = discr.dim flux_tpair = TracePair(cv_tpair.dd, interior=inviscid_flux(discr, eos, cv_tpair.int), exterior=inviscid_flux(discr, eos, cv_tpair.ext)) lam = actx.np.maximum( - compute_wavespeed(dim, eos=eos, cv=cv_tpair.int), - compute_wavespeed(dim, eos=eos, cv=cv_tpair.ext) + compute_wavespeed(eos=eos, cv=cv_tpair.int), + compute_wavespeed(eos=eos, cv=cv_tpair.ext) ) normal = thaw(actx, discr.normal(cv_tpair.dd)) # todo: user-supplied flux routine - # flux_weak = make_conserved( - # dim, q=lfr_flux(cv_tpair, flux_tpair, normal=normal, lam=lam) - # ) flux_weak = lfr_flux(cv_tpair, flux_tpair, normal=normal, lam=lam) if local is False: From b1eb999e9d2f3b13ce0521876e8ee2a48121b4e7 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 9 Jun 2021 13:53:34 -0500 Subject: [PATCH 140/385] Update to latest (and greatest) API from grudge --- mirgecom/inviscid.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 74c4510b3..8a86d08ff 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -98,12 +98,14 @@ def get_inviscid_timestep(discr, eos, q): class:`~meshmode.dof_array.DOFArray` The maximum stable timestep at each node. """ - from grudge.dt_utils import estimate_local_timestep + from grudge.dt_utils import characteristic_length_scales from mirgecom.fluid import compute_wavespeed dim = discr.dim cv = split_conserved(dim, q) - return estimate_local_timestep(discr, compute_wavespeed(dim, eos, cv)) + return ( + characteristic_length_scales(discr)/compute_wavespeed(discr, eos, cv) + ) def get_inviscid_cfl(discr, eos, dt, q): From 6ef0f0557fda7635550f59bf491b43d258dcc6d1 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 9 Jun 2021 14:15:58 -0500 Subject: [PATCH 141/385] Fix transcribo on grudge dt utils func name --- mirgecom/inviscid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 8a86d08ff..43a89347f 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -98,13 +98,13 @@ def get_inviscid_timestep(discr, eos, q): class:`~meshmode.dof_array.DOFArray` The maximum stable timestep at each node. """ - from grudge.dt_utils import characteristic_length_scales + from grudge.dt_utils import characteristic_lengthscales from mirgecom.fluid import compute_wavespeed dim = discr.dim cv = split_conserved(dim, q) return ( - characteristic_length_scales(discr)/compute_wavespeed(discr, eos, cv) + characteristic_lengthscales(discr)/compute_wavespeed(discr, eos, cv) ) From 0c2e986abd136b9bdbe6a93de7c6b44df67eba2a Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 10 Jun 2021 23:17:47 -0500 Subject: [PATCH 142/385] Update simutils/health/checkpoint - move most to driver, simplify --- examples/mixture-mpi.py | 58 +++++- mirgecom/simutil.py | 406 +++++++++++++++++++--------------------- 2 files changed, 242 insertions(+), 222 deletions(-) diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 28ee4033b..f9197f164 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -39,7 +39,6 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, - sim_checkpoint, generate_and_distribute_mesh ) from mirgecom.io import make_init_message @@ -77,6 +76,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): current_t = 0 constant_cfl = False nstatus = 1 + nhealth = 1 nviz = 1 rank = 0 checkpoint_t = current_t @@ -150,10 +150,58 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - sim_checkpoint(discr, visualizer, eos, q=state, - exact_soln=initializer, vizname=casename, step=step, - t=t, dt=dt, nstatus=nstatus, nviz=nviz, - exittol=exittol, constant_cfl=constant_cfl) + from mirgecom.simutils import check_step + do_status = check_step(step=step, interval=nstatus) + do_viz = check_step(step=step, interval=nviz) + do_health = check_step(step=step, interval=nhealth) + + if do_status or do_viz or do_health: + from mirgecom.simutil import compare_fluid_solutions + dv = eos.dependent_vars(state) + exact_mix = initializer(x_vec=nodes, eos=eos, t=t) + component_errors = compare_fluid_solutions(state, exact_mix) + resid = state - exact_mix + io_fields = [ + ("cv", state), + ("dv", dv), + ("exact_mix", exact_mix), + ("resid", resid) + ] + + if do_status: # This is bad, logging already completely replaces this + from mirgecom.io import make_status_message + status_msg = make_status_message(discr, t=t, step=step, dt=dt, + cfl=current_cfl, dependent_vars=dv) + status_msg += ( + "\n------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + if rank == 0: + logger.info(status_msg) + + if do_health: + from mirgecom.simutils import check_naninf_local, check_negative_local + if check_naninf_local(discr, "vol", dv.pressure) \ + or check_negative_local(discr, "vol", dv.pressure): + errors = 1 + message = "Invalid pressure data found.\n" + if np.max(component_errors) > exittol: + errors = errors + 1 + message += "Solution errors exceed tolerance.\n" + errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) + if errors > 0: + if rank == 0: + logger.info("Fluid solution failed health check.") + logger.info(message) # do this on all ranks + + if do_viz or errors > 0: + from mirgecom.simutils import sim_visualization + sim_visualization(discr, io_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + if errors > 0: + a = 1/0 + print(f"{a=}") + return state current_step, current_t, current_state = \ diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index c5222576a..9fd01add0 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -47,10 +47,11 @@ import logging import numpy as np -from meshmode.dof_array import thaw -from mirgecom.io import make_status_message +# from meshmode.dof_array import thaw +# from mirgecom.io import make_status_message from mirgecom.inviscid import get_inviscid_timestep # bad smell? -from mirgecom.exceptions import SynchronizedError, SimulationHealthError +from mirgecom.exceptions import SimulationHealthError +import grudge.op as op logger = logging.getLogger(__name__) @@ -88,52 +89,35 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, return mydt -def sim_visualization(discr, eos, state, visualizer, vizname, - step=0, t=0, - exact_soln=None, viz_fields=None, - overwrite=False, vis_timer=None): +def sim_visualization(discr, io_fields, visualizer, vizname, + step=0, t=0, overwrite=False, vis_timer=None): """Visualize the simulation fields. - Write VTK output of the conserved state and and specified derived - quantities *viz_fields*. + Write VTK output for the specified fields. Parameters ---------- - eos: mirgecom.eos.GasEOS - Implementing the pressure and temperature functions for - returning pressure and temperature as a function of the state. - state - State array which expects at least the conserved quantities - (mass, energy, momentum) for the fluid at each point. For multi-component - fluids, the conserved quantities should include - (mass, energy, momentum, species_mass), where *species_mass* is a vector - of species masses. visualizer: A :class:`meshmode.discretization.visualization.Visualizer` VTK output object. + io_fields: + List of tuples indicating the (name, data) for each field to write. """ from contextlib import nullcontext - from mirgecom.fluid import split_conserved from mirgecom.io import make_rank_fname, make_par_fname - cv = split_conserved(discr.dim, state) - dependent_vars = eos.dependent_vars(cv) - - io_fields = [ - ("cv", cv), - ("dv", dependent_vars) - ] - if exact_soln is not None: - actx = cv.mass.array_context - nodes = thaw(actx, discr.nodes()) - expected_state = exact_soln(x_vec=nodes, t=t, eos=eos) - exact_list = [ - ("exact_soln", expected_state), - ] - io_fields.extend(exact_list) - - if viz_fields is not None: - io_fields.extend(viz_fields) + # io_fields = [ + # ("cv", cv), + # ("dv", dependent_vars) + # ] + # if exact_soln is not None: + # exact_list = [ + # ("exact_soln", exact_soln), + # ] + # io_fields.extend(exact_list) + + # if viz_fields is not None: + # io_fields.extend(viz_fields) comm = discr.mpi_communicator rank = 0 @@ -157,95 +141,106 @@ def sim_visualization(discr, eos, state, visualizer, vizname, ) -def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, - step=0, t=0, dt=0, cfl=1.0, nstatus=-1, nviz=-1, exittol=1e-16, - constant_cfl=False, viz_fields=None, overwrite=False, - vis_timer=None): - """Checkpoint the simulation status. - - Checkpoints the simulation status by reporting relevant diagnostic - quantities, such as pressure/temperature, and visualization. - - Parameters - ---------- - eos: mirgecom.eos.GasEOS - Implementing the pressure and temperature functions for - returning pressure and temperature as a function of the state *q*. - q - State array which expects at least the conserved quantities - (mass, energy, momentum) for the fluid at each point. For multi-component - fluids, the conserved quantities should include - (mass, energy, momentum, species_mass), where *species_mass* is a vector - of species masses. - nstatus: int - An integer denoting the step frequency for performing status checks. - nviz: int - An integer denoting the step frequency for writing vtk output. - """ - exception = None - comm = discr.mpi_communicator - rank = 0 - if comm is not None: - rank = comm.Get_rank() - - try: - # Status checks - if check_step(step=step, interval=nstatus): - from mirgecom.fluid import split_conserved - - # if constant_cfl is False: - # current_cfl = get_inviscid_cfl(discr=discr, q=q, - # eos=eos, dt=dt) - - cv = split_conserved(discr.dim, q) - dependent_vars = eos.dependent_vars(cv) - msg = make_status_message(discr=discr, - t=t, step=step, dt=dt, cfl=cfl, - dependent_vars=dependent_vars) - if rank == 0: - logger.info(msg) - - # Check the health of the simulation - sim_healthcheck(discr, eos, q, step=step, t=t) - - # Compare with exact solution, if provided - if exact_soln is not None: - compare_with_analytic_solution( - discr, eos, q, exact_soln, - step=step, t=t, exittol=exittol - ) - - # Visualization - if check_step(step=step, interval=nviz): - sim_visualization( - discr, eos, q, visualizer, vizname, - step=step, t=t, - exact_soln=exact_soln, viz_fields=viz_fields, - overwrite=overwrite, vis_timer=vis_timer - ) - except SynchronizedError as err: - exception = err - - terminate = True if exception is not None else False - - if comm is not None: - from mpi4py import MPI - - terminate = comm.allreduce(terminate, MPI.LOR) - - if terminate: - if rank == 0: - logger.info("Visualizing crashed state ...") - - # Write out crashed field - sim_visualization(discr, eos, q, - visualizer, vizname=vizname, - step=step, t=t, - viz_fields=viz_fields) - raise exception - - -def sim_healthcheck(discr, eos, state, step=0, t=0): +# def sim_checkpoint(discr, visualizer, eos, q, vizname, exact_soln=None, +# step=0, t=0, dt=0, cfl=1.0, nstatus=-1, nviz=-1, exittol=1e-16, +# constant_cfl=False, viz_fields=None, overwrite=False, +# vis_timer=None): +# """Checkpoint the simulation status. +# +# Checkpoints the simulation status by reporting relevant diagnostic +# quantities, such as pressure/temperature, and visualization. +# +# Parameters +# ---------- +# eos: mirgecom.eos.GasEOS +# Implementing the pressure and temperature functions for +# returning pressure and temperature as a function of the state *q*. +# q +# State array which expects at least the conserved quantities +# (mass, energy, momentum) for the fluid at each point. For multi-component +# fluids, the conserved quantities should include +# (mass, energy, momentum, species_mass), where *species_mass* is a vector +# of species masses. +# nstatus: int +# An integer denoting the step frequency for performing status checks. +# nviz: int +# An integer denoting the step frequency for writing vtk output. +# """ +# exception = None +# comm = discr.mpi_communicator +# rank = 0 +# if comm is not None: +# rank = comm.Get_rank() +# +# try: +# # Status checks +# if check_step(step=step, interval=nstatus): +# from mirgecom.fluid import split_conserved +# +# # if constant_cfl is False: +# # current_cfl = get_inviscid_cfl(discr=discr, q=q, +# # eos=eos, dt=dt) +# +# cv = split_conserved(discr.dim, q) +# dependent_vars = eos.dependent_vars(cv) +# msg = make_status_message(discr=discr, +# # t=t, step=step, dt=dt, cfl=cfl, +# dependent_vars=dependent_vars) +# if rank == 0: +# logger.info(msg) +# +# # Check the health of the simulation +# sim_healthcheck(discr, eos, q, step=step, t=t) +# +# # Compare with exact solution, if provided +# if exact_soln is not None: +# compare_with_analytic_solution( +# discr, eos, q, exact_soln, +# step=step, t=t, exittol=exittol +# ) +# +# # Visualization +# if check_step(step=step, interval=nviz): +# sim_visualization( +# discr, eos, q, visualizer, vizname, +# step=step, t=t, +# exact_soln=exact_soln, viz_fields=viz_fields, +# overwrite=overwrite, vis_timer=vis_timer +# ) +# except SynchronizedError as err: +# exception = err +# +# terminate = True if exception is not None else False +# +# if comm is not None: +# from mpi4py import MPI +# +# terminate = comm.allreduce(terminate, MPI.LOR) +# +# if terminate: +# if rank == 0: +# logger.info("Visualizing crashed state ...") +# +# # Write out crashed field +# sim_visualization(discr, eos, q, +# visualizer, vizname=vizname, +# step=step, t=t, +# viz_fields=viz_fields) +# raise exception + + +def check_negative_local(discr, dd, field): + """Check for any negative values.""" + return op.nodal_min_loc(discr, dd, field) < 0 + + +def check_naninf_local(discr, dd, field): + """Check for any NANs or Infs in the field.""" + s = op.nodal_sum_local(discr, dd, field) + return np.isnan(s) or (s == np.inf) + + +def sim_healthcheck(discr, eos, cv): """Check the global health of the fluids state. Determine the health of a state by inspecting for unphyiscal @@ -256,100 +251,77 @@ def sim_healthcheck(discr, eos, state, step=0, t=0): eos: mirgecom.eos.GasEOS Implementing the pressure and temperature functions for returning pressure and temperature as a function of the state. - state - State array which expects at least the conserved quantities - (mass, energy, momentum) for the fluid at each point. For multi-component - fluids, the conserved quantities should include - (mass, energy, momentum, species_mass), where *species_mass* is a vector - of species masses. + cv: :class:`~mirgecom.fluid.ConservedVars` + The fluid conserved variables """ - from mirgecom.fluid import split_conserved - import grudge.op as op - # NOTE: Derived quantities are functions of the conserved variables. # Therefore is it sufficient to check for unphysical values of - # temperature and pressure. - cv = split_conserved(discr.dim, state) - dependent_vars = eos.dependent_vars(cv) - pressure = dependent_vars.pressure - temperature = dependent_vars.temperature - - # Check for NaN - if (np.isnan(op.nodal_sum_loc(discr, "vol", pressure)) - or np.isnan(op.nodal_sum_loc(discr, "vol", temperature))): - raise SimulationHealthError( - message=("Simulation exited abnormally; detected a NaN.") - ) - - # Check for non-positivity - if (op.nodal_min_loc(discr, "vol", pressure) < 0 - or op.nodal_min_loc(discr, "vol", temperature) < 0): - raise SimulationHealthError( - message=("Simulation exited abnormally; " - "found non-positive values for pressure or temperature.") - ) - - # Check for blow-up - if (op.nodal_sum_loc(discr, "vol", pressure) == np.inf - or op.nodal_sum_loc(discr, "vol", temperature) == np.inf): - raise SimulationHealthError( - message=("Simulation exited abnormally; " - "derived quantities are not finite.") - ) - - -def compare_with_analytic_solution(discr, eos, state, exact_soln, - step=0, t=0, exittol=None): - """Compute the infinite norm of the problem residual. - - Computes the infinite norm of the residual with respect to a specified - exact solution *exact_soln*. If the error is larger than *exittol*, - raises a :class:`mirgecom.exceptions.SimulationHealthError`. - - Parameters - ---------- - eos: mirgecom.eos.GasEOS - Implementing the pressure and temperature functions for - returning pressure and temperature as a function of the state. - state - State array which expects at least the conserved quantities - (mass, energy, momentum) for the fluid at each point. For multi-component - fluids, the conserved quantities should include - (mass, energy, momentum, species_mass), where *species_mass* is a vector - of species masses. - exact_soln: - A callable for the exact solution with signature: - ``exact_soln(x_vec, t, eos)`` where `x_vec` are the nodes, - `t` is time, and `eos` is a :class:`mirgecom.eos.GasEOS`. - """ - if exittol is None: - exittol = 1e-16 - - actx = discr._setup_actx - nodes = thaw(actx, discr.nodes()) - expected_state = exact_soln(x_vec=nodes, t=t, eos=eos) - exp_resid = state - expected_state - norms = [discr.norm(v, np.inf) for v in exp_resid] - - comm = discr.mpi_communicator - rank = 0 - if comm is not None: - rank = comm.Get_rank() - - statusmesg = ( - f"Errors: {step=} {t=}\n" - f"------- errors = " - + ", ".join("%.3g" % err_norm for err_norm in norms) - ) - - if rank == 0: - logger.info(statusmesg) - - if max(norms) > exittol: - raise SimulationHealthError( - message=("Simulation exited abnormally; " - "solution doesn't agree with analytic result.") - ) + # pressure. Temperature is not needed. + from mpi4py import MPI + errors = 0 + pressure = eos.pressure(cv) + if check_naninf_local(discr, "vol", pressure) \ + or check_negative_local(discr, "vol", pressure): + errors = 1 + message = "Invalid data found in dependent variables." + if discr.mpi_communicator.allreduce(errors, op=MPI.SUM) > 0: + raise SimulationHealthError(message=message) + + +def compare_fluid_solutions(discr, red_state, blue_state): + """Return inf norm of (*red_state* - *blue_state*) for each component.""" + resid = red_state - blue_state + return [discr.norm(v, np.inf) for v in resid.join()] + + +# def compare_with_analytic_solution(discr, eos, state, exact_soln, +# step=0, t=0, exittol=None): +# """Compute the infinite norm of the problem residual. +# +# Computes the infinite norm of the residual with respect to a specified +# exact solution *exact_soln*. If the error is larger than *exittol*, +# raises a :class:`mirgecom.exceptions.SimulationHealthError`. +# +# Parameters +# ---------- +# eos: mirgecom.eos.GasEOS +# Implementing the pressure and temperature functions for +# returning pressure and temperature as a function of the state. +# state: :class:`ConservedVars` +# Fluid solution +# exact_soln: +# A callable for the exact solution with signature: +# ``exact_soln(x_vec, eos, t)`` where `x_vec` are the nodes, +# `t` is time, and `eos` is a :class:`mirgecom.eos.GasEOS`. +# """ +# if exittol is None: +# exittol = 1e-16 + +# actx = discr._setup_actx +# nodes = thaw(actx, discr.nodes()) +# expected_state = exact_soln(x_vec=nodes, t=t, eos=eos) +# exp_resid = state - expected_state +# norms = [discr.norm(v, np.inf) for v in exp_resid] + +# comm = discr.mpi_communicator +# rank = 0 +# if comm is not None: +# rank = comm.Get_rank() + +# statusmesg = ( +# f"Errors: {step=} {t=}\n" +# f"------- errors = " +# + ", ".join("%.3g" % err_norm for err_norm in norms) +# ) + +# if rank == 0: +# logger.info(statusmesg) + +# if max(norms) > exittol: +# raise SimulationHealthError( +# message=("Simulation exited abnormally; " +# "solution doesn't agree with analytic result.") +# ) def generate_and_distribute_mesh(comm, generate_mesh): From 15bdf3d7eb9e542865c8167f3e85b8b22b0aea40 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 11 Jun 2021 01:24:46 -0500 Subject: [PATCH 143/385] Port examples into place, slight adjustment/cleanup/corrections for simutils interfaces. --- examples/autoignition-mpi.py | 56 +++++++++++++++++++++++++++---- examples/lump-mpi.py | 64 +++++++++++++++++++++++++++++++++--- examples/mixture-mpi.py | 13 ++++---- examples/pulse-mpi.py | 45 ++++++++++++++++++++++++- examples/scalar-lump-mpi.py | 54 ++++++++++++++++++++++++++++++ examples/sod-mpi.py | 55 +++++++++++++++++++++++++++++++ examples/vortex-mpi.py | 54 ++++++++++++++++++++++++++++-- mirgecom/simutil.py | 16 ++++----- 8 files changed, 328 insertions(+), 29 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 148cef495..3f033339b 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -80,6 +80,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 1 nviz = 5 + nhealth = 1 rank = 0 checkpoint_t = current_t current_step = 0 @@ -221,15 +222,56 @@ def my_rhs(t, state): + eos.get_species_source_terms(state)) def my_checkpoint(step, t, dt, state): - reaction_rates = eos.get_production_rates(state) - viz_fields = [("reaction_rates", reaction_rates)] - print(f"{viz_fields=}") - - (current_step, current_t, current_state) = \ + from mirgecom.simutil import check_step + do_status = check_step(step=step, interval=nstatus) + do_viz = check_step(step=step, interval=nviz) + do_health = check_step(step=step, interval=nhealth) + + if do_status or do_viz or do_health: + dv = eos.dependent_vars(state) + reaction_rates = eos.get_production_rates(state) + io_fields = [ + ("cv", state), + ("dv", dv), + ("reaction_rates", reaction_rates) + ] + + if do_status: # This is bad, logging already completely replaces this + from mirgecom.io import make_status_message + status_msg = make_status_message(discr=discr, t=t, step=step, dt=dt, + cfl=current_cfl, dependent_vars=dv) + if rank == 0: + logger.info(status_msg) + + errors = 0 + if do_health: + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure) \ + or check_range_local(discr, "vol", dv.pressure): + errors = 1 + message = "Invalid pressure data found.\n" + errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) + if errors > 0: + if rank == 0: + logger.info("Fluid solution failed health check.") + logger.info(message) # do this on all ranks + + if do_viz or errors > 0: + from mirgecom.simutil import sim_visualization + sim_visualization(discr, io_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + if errors > 0: + a = 1/0 + print(f"{a=}") + + return state + + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=my_checkpoint, + pre_step_callback=my_checkpoint, get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) + t=current_t, t_final=t_final, eos=eos, dim=dim) if not check_step(current_step, nviz): # If final step not an output step if rank == 0: diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index da60d46a8..231b9b2fe 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -79,6 +79,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} constant_cfl = False nstatus = 1 + nhealth = 1 nviz = 1 rank = 0 checkpoint_t = current_t @@ -129,13 +130,66 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - print(f"{step=}") - - (current_step, current_t, current_state) = \ + from mirgecom.simutil import check_step + do_status = check_step(step=step, interval=nstatus) + do_viz = check_step(step=step, interval=nviz) + do_health = check_step(step=step, interval=nhealth) + + if do_status or do_viz or do_health: + from mirgecom.simutil import compare_fluid_solutions + dv = eos.dependent_vars(state) + exact_mix = initializer(x_vec=nodes, eos=eos, t=t) + component_errors = compare_fluid_solutions(discr, state, exact_mix) + resid = state - exact_mix + io_fields = [ + ("cv", state), + ("dv", dv), + ("exact_mix", exact_mix), + ("resid", resid) + ] + + if do_status: # This is bad, logging already completely replaces this + from mirgecom.io import make_status_message + status_msg = make_status_message(discr=discr, t=t, step=step, dt=dt, + cfl=current_cfl, dependent_vars=dv) + status_msg += ( + "\n------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + if rank == 0: + logger.info(status_msg) + + errors = 0 + if do_health: + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure) \ + or check_range_local(discr, "vol", dv.pressure): + errors = 1 + message = "Invalid pressure data found.\n" + if np.max(component_errors) > exittol: + errors = errors + 1 + message += "Solution errors exceed tolerance.\n" + errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) + if errors > 0: + if rank == 0: + logger.info("Fluid solution failed health check.") + logger.info(message) # do this on all ranks + + if do_viz or errors > 0: + from mirgecom.simutil import sim_visualization + sim_visualization(discr, io_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + if errors > 0: + a = 1/0 + print(f"{a=}") + + return state + + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=my_checkpoint, + pre_step_callback=my_checkpoint, get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) + t=current_t, t_final=t_final, eos=eos, dim=dim) # if current_t != checkpoint_t: if rank == 0: diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index c19751907..a3480fffc 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -150,7 +150,7 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): - from mirgecom.simutils import check_step + from mirgecom.simutil import check_step do_status = check_step(step=step, interval=nstatus) do_viz = check_step(step=step, interval=nviz) do_health = check_step(step=step, interval=nhealth) @@ -159,7 +159,7 @@ def my_checkpoint(step, t, dt, state): from mirgecom.simutil import compare_fluid_solutions dv = eos.dependent_vars(state) exact_mix = initializer(x_vec=nodes, eos=eos, t=t) - component_errors = compare_fluid_solutions(state, exact_mix) + component_errors = compare_fluid_solutions(discr, state, exact_mix) resid = state - exact_mix io_fields = [ ("cv", state), @@ -170,7 +170,7 @@ def my_checkpoint(step, t, dt, state): if do_status: # This is bad, logging already completely replaces this from mirgecom.io import make_status_message - status_msg = make_status_message(discr, t=t, step=step, dt=dt, + status_msg = make_status_message(discr=discr, t=t, step=step, dt=dt, cfl=current_cfl, dependent_vars=dv) status_msg += ( "\n------- errors=" @@ -178,10 +178,11 @@ def my_checkpoint(step, t, dt, state): if rank == 0: logger.info(status_msg) + errors = 0 if do_health: - from mirgecom.simutils import check_naninf_local, check_negative_local + from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ - or check_negative_local(discr, "vol", dv.pressure): + or check_range_local(discr, "vol", dv.pressure): errors = 1 message = "Invalid pressure data found.\n" if np.max(component_errors) > exittol: @@ -194,7 +195,7 @@ def my_checkpoint(step, t, dt, state): logger.info(message) # do this on all ranks if do_viz or errors > 0: - from mirgecom.simutils import sim_visualization + from mirgecom.simutil import sim_visualization sim_visualization(discr, io_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index b5f3e41a4..49c82163a 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -84,8 +84,9 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): wall = AdiabaticSlipBoundary() boundaries = {BTAG_ALL: wall} constant_cfl = False - nstatus = 10 + nstatus = 1 nviz = 10 + nhealth = 1 rank = 0 checkpoint_t = current_t current_step = 0 @@ -149,6 +150,48 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): + from mirgecom.simutil import check_step + do_status = check_step(step=step, interval=nstatus) + do_viz = check_step(step=step, interval=nviz) + do_health = check_step(step=step, interval=nhealth) + + if do_status or do_viz or do_health: + pressure = eos.pressure(state) + io_fields = [ + ("cv", state), + ("pressure", pressure) + ] + + if do_status: # This is bad, logging already completely replaces this + from mirgecom.io import make_status_message + dv = eos.dependent_vars(state) + status_msg = make_status_message(discr=discr, t=t, step=step, dt=dt, + cfl=current_cfl, dependent_vars=dv) + if rank == 0: + logger.info(status_msg) + + errors = 0 + if do_health: + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", pressure) \ + or check_range_local(discr, "vol", pressure): + errors = 1 + message = "Invalid pressure data found.\n" + errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) + if errors > 0: + if rank == 0: + logger.info("Fluid solution failed health check.") + logger.info(message) # do this on all ranks + + if do_viz or errors > 0: + from mirgecom.simutil import sim_visualization + sim_visualization(discr, io_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + if errors > 0: + a = 1/0 + print(f"{a=}") + return state current_step, current_t, current_state = \ diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 03744e541..0bc82a453 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -74,6 +74,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 1 nviz = 1 + nhealth = 1 rank = 0 checkpoint_t = current_t current_step = 0 @@ -139,6 +140,59 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): + from mirgecom.simutil import check_step + do_status = check_step(step=step, interval=nstatus) + do_viz = check_step(step=step, interval=nviz) + do_health = check_step(step=step, interval=nhealth) + + if do_status or do_viz or do_health: + from mirgecom.simutil import compare_fluid_solutions + dv = eos.dependent_vars(state) + exact_mix = initializer(x_vec=nodes, eos=eos, t=t) + component_errors = compare_fluid_solutions(discr, state, exact_mix) + resid = state - exact_mix + io_fields = [ + ("cv", state), + ("dv", dv), + ("exact_mix", exact_mix), + ("resid", resid) + ] + + if do_status: # This is bad, logging already completely replaces this + from mirgecom.io import make_status_message + status_msg = make_status_message(discr=discr, t=t, step=step, dt=dt, + cfl=current_cfl, dependent_vars=dv) + status_msg += ( + "\n------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + if rank == 0: + logger.info(status_msg) + + errors = 0 + if do_health: + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure) \ + or check_range_local(discr, "vol", dv.pressure): + errors = 1 + message = "Invalid pressure data found.\n" + if np.max(component_errors) > exittol: + errors = errors + 1 + message += "Solution errors exceed tolerance.\n" + errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) + if errors > 0: + if rank == 0: + logger.info("Fluid solution failed health check.") + logger.info(message) # do this on all ranks + + if do_viz or errors > 0: + from mirgecom.simutil import sim_visualization + sim_visualization(discr, io_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + if errors > 0: + a = 1/0 + print(f"{a=}") + return state current_step, current_t, current_state = \ diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 6f74545a6..205afaeeb 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -24,6 +24,7 @@ THE SOFTWARE. """ import logging +import numpy as np import pyopencl as cl import pyopencl.tools as cl_tools from functools import partial @@ -77,6 +78,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 10 nviz = 10 + nhealth = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -128,6 +130,59 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): + from mirgecom.simutil import check_step + do_status = check_step(step=step, interval=nstatus) + do_viz = check_step(step=step, interval=nviz) + do_health = check_step(step=step, interval=nhealth) + + if do_status or do_viz or do_health: + from mirgecom.simutil import compare_fluid_solutions + dv = eos.dependent_vars(state) + sod_exact = initializer(x_vec=nodes, eos=eos, t=t) + component_errors = compare_fluid_solutions(discr, state, sod_exact) + resid = state - sod_exact + io_fields = [ + ("cv", state), + ("dv", dv), + ("sod_exact", sod_exact), + ("resid", resid) + ] + + if do_status: # This is bad, logging already completely replaces this + from mirgecom.io import make_status_message + status_msg = make_status_message(discr=discr, t=t, step=step, dt=dt, + cfl=current_cfl, dependent_vars=dv) + status_msg += ( + "\n------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + if rank == 0: + logger.info(status_msg) + + errors = 0 + if do_health: + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure) \ + or check_range_local(discr, "vol", dv.pressure): + errors = 1 + message = "Invalid pressure data found.\n" + if np.max(component_errors) > exittol: + errors = errors + 1 + message += "Solution errors exceed tolerance.\n" + errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) + if errors > 0: + if rank == 0: + logger.info("Fluid solution failed health check.") + logger.info(message) # do this on all ranks + + if do_viz or errors > 0: + from mirgecom.simutil import sim_visualization + sim_visualization(discr, io_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + if errors > 0: + a = 1/0 + print(f"{a=}") + return state current_step, current_t, current_state = \ diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index a973ecf3a..4438bbf52 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -102,6 +102,7 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal constant_cfl = False nstatus = 10 nviz = 10 + nhealth = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -175,14 +176,63 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): + from mirgecom.simutil import check_step + do_status = check_step(step=step, interval=nstatus) + do_viz = check_step(step=step, interval=nviz) + do_health = check_step(step=step, interval=nhealth) + + if do_status or do_viz or do_health: + from mirgecom.simutil import compare_fluid_solutions + dv = eos.dependent_vars(state) + vortex_exact = initializer(x_vec=nodes, eos=eos, t=t) + component_errors = compare_fluid_solutions(discr, state, vortex_exact) + resid = state - vortex_exact + io_fields = [ + ("cv", state), + ("dv", dv), + ("vortex_exact", vortex_exact), + ("resid", resid) + ] + + if do_status: + status_msg = ( + "\n------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + if rank == 0: + logger.info(status_msg) + + errors = 0 + if do_health: + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure) \ + or check_range_local(discr, "vol", dv.pressure): + errors = 1 + message = "Invalid pressure data found.\n" + if np.max(component_errors) > exittol: + errors = errors + 1 + message += "Solution errors exceed tolerance.\n" + errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) + if errors > 0: + if rank == 0: + logger.info("Fluid solution failed health check.") + logger.info(message) # do this on all ranks + + if do_viz or errors > 0: + from mirgecom.simutil import sim_visualization + sim_visualization(discr, io_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + if errors > 0: + a = 1/0 + print(f"{a=}") + return state current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_checkpoint, get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final, - logmgr=logmgr, eos=eos, dim=dim) + t=current_t, t_final=t_final, eos=eos, dim=dim) if rank == 0: logger.info("Checkpointing final state ...") diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index b253e1ce8..6df9088c3 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -5,12 +5,14 @@ .. autofunction:: check_step .. autofunction:: inviscid_sim_timestep +.. autofunction:: sim_visualization -Diagnostic callbacks +Diagnostic utilities -------------------- -.. autofunction:: sim_visualization -.. autofunction:: compare_with_analytic_solution +.. autofunction:: compare_fluid_solutions +.. autofunction:: check_naninf_local +.. autofunction:: check_range_local Mesh utilities -------------- @@ -45,8 +47,6 @@ import logging import numpy as np -# from meshmode.dof_array import thaw -# from mirgecom.io import make_status_message from mirgecom.inviscid import get_inviscid_timestep # bad smell? import grudge.op as op @@ -128,14 +128,14 @@ def sim_visualization(discr, io_fields, visualizer, vizname, def check_range_local(discr, dd, field, min_value=0, max_value=np.inf): """Check for any negative values.""" return ( - op.nodal_min_local(discr, dd, field) < min_value - or op.nodal_max_local(discr, dd, field) > max_value + op.nodal_min_loc(discr, dd, field) < min_value + or op.nodal_max_loc(discr, dd, field) > max_value ) def check_naninf_local(discr, dd, field): """Check for any NANs or Infs in the field.""" - s = op.nodal_sum_local(discr, dd, field) + s = op.nodal_sum_loc(discr, dd, field) return np.isnan(s) or (s == np.inf) From 68302b4b3018b3810d29efc7e0e4466aab454cb1 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 11 Jun 2021 01:56:20 -0500 Subject: [PATCH 144/385] Update tests to match the facilities. --- test/{test_callbacks.py => test_simutil.py} | 53 +++++++++++++-------- 1 file changed, 32 insertions(+), 21 deletions(-) rename test/{test_callbacks.py => test_simutil.py} (66%) diff --git a/test/test_callbacks.py b/test/test_simutil.py similarity index 66% rename from test/test_callbacks.py rename to test/test_simutil.py index 7ed1b9663..23ea148ea 100644 --- a/test/test_callbacks.py +++ b/test/test_simutil.py @@ -25,7 +25,7 @@ """ import numpy as np -import pytest +import pytest # noqa from arraycontext import ( # noqa thaw, @@ -33,15 +33,14 @@ as pytest_generate_tests ) -from mirgecom.fluid import join_conserved +from mirgecom.fluid import make_conserved from mirgecom.eos import IdealSingleGas from grudge.eager import EagerDGDiscretization def test_basic_cfd_healthcheck(actx_factory): - from mirgecom.simutil import sim_healthcheck - + """Quick test of some health checking utilities.""" actx = actx_factory() nel_1d = 4 dim = 2 @@ -65,12 +64,10 @@ def test_basic_cfd_healthcheck(actx_factory): mom = mass * velocity eos = IdealSingleGas() - q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) - - from mirgecom.exceptions import SimulationHealthError + cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) - with pytest.raises(SimulationHealthError): - sim_healthcheck(discr, eos, q) + from mirgecom.simutil import check_range_local + assert check_range_local(discr, "vol", mass) # Let's make another very bad state (nans) mass = 1*ones @@ -78,10 +75,11 @@ def test_basic_cfd_healthcheck(actx_factory): velocity = np.nan * nodes mom = mass * velocity - q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) + cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) + pressure = eos.pressure(cv) - with pytest.raises(SimulationHealthError): - sim_healthcheck(discr, eos, q) + from mirgecom.simutil import check_naninf_local + assert check_naninf_local(discr, "vol", pressure) # Let's make one last very bad state (inf) mass = 1*ones @@ -89,15 +87,24 @@ def test_basic_cfd_healthcheck(actx_factory): velocity = 2 * nodes mom = mass * velocity - q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) + cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) + pressure = eos.pressure(cv) + + assert check_naninf_local(discr, "vol", pressure) - with pytest.raises(SimulationHealthError): - sim_healthcheck(discr, eos, q) + # What the hey, test a good one + energy = 2.5 + .5*np.dot(mom, mom) + cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) + pressure = eos.pressure(cv) + + assert not check_naninf_local(discr, "vol", pressure) + assert not check_range_local(discr, "vol", pressure) def test_analytic_comparison(actx_factory): + """Quick test of state comparison routine.""" from mirgecom.initializers import Vortex2D - from mirgecom.simutil import compare_with_analytic_solution + from mirgecom.simutil import compare_fluid_solutions actx = actx_factory() nel_1d = 4 @@ -114,15 +121,19 @@ def test_analytic_comparison(actx_factory): nodes = thaw(discr.nodes(), actx) zeros = discr.zeros(actx) ones = zeros + 1.0 - eos = IdealSingleGas() mass = ones energy = ones velocity = 2 * nodes mom = mass * velocity + vortex_init = Vortex2D() + vortex_soln = vortex_init(x_vec=nodes, eos=IdealSingleGas()) - q = join_conserved(dim, mass=mass, energy=energy, momentum=mom) + cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) + resid = vortex_soln - cv + expected_errors = [discr.norm(v, np.inf) for v in resid.join()] - from mirgecom.exceptions import SimulationHealthError + errors = compare_fluid_solutions(discr, cv, cv) + assert max(errors) == 0 - with pytest.raises(SimulationHealthError): - compare_with_analytic_solution(discr, eos, q, Vortex2D()) + errors = compare_fluid_solutions(discr, cv, vortex_soln) + assert errors == expected_errors From 8b506e166232506971525a53b33b4945fb769914 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 11 Jun 2021 03:01:02 -0500 Subject: [PATCH 145/385] Twiddle to test both pressure and mass for first bad state. --- test/test_simutil.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_simutil.py b/test/test_simutil.py index 23ea148ea..572504bf2 100644 --- a/test/test_simutil.py +++ b/test/test_simutil.py @@ -59,15 +59,17 @@ def test_basic_cfd_healthcheck(actx_factory): # Let's make a very bad state (negative mass) mass = -1*ones - energy = zeros + 2.5 velocity = 2 * nodes mom = mass * velocity + energy = zeros + .5*np.dot(mom, mom)/mass eos = IdealSingleGas() cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) + pressure = eos.pressure(cv) from mirgecom.simutil import check_range_local assert check_range_local(discr, "vol", mass) + assert check_range_local(discr, "vol", pressure, min_value=1e-6) # Let's make another very bad state (nans) mass = 1*ones From 51ade9b1e6f6561c46097e03b56ee2cf1ea71e3a Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 11 Jun 2021 13:50:09 -0500 Subject: [PATCH 146/385] Sharpen the error sync. Co-authored-by: Thomas H. Gibson --- examples/autoignition-mpi.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 3f033339b..714855909 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -243,15 +243,17 @@ def my_checkpoint(step, t, dt, state): if rank == 0: logger.info(status_msg) - errors = 0 + errored = False if do_health: from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ or check_range_local(discr, "vol", dv.pressure): - errors = 1 + errored = True message = "Invalid pressure data found.\n" - errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) - if errors > 0: + comm = discr.mpi_communicator + if comm is not None: + errored = comm.allreduce(errored, op=MPI.LOR) + if errored: if rank == 0: logger.info("Fluid solution failed health check.") logger.info(message) # do this on all ranks From 68339029d48c879a3df883c27959a03657bde0a8 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 11 Jun 2021 13:52:42 -0500 Subject: [PATCH 147/385] Update error handling to match review suggestion by @thomasgibson. --- examples/autoignition-mpi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 714855909..3f6f089a5 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -258,12 +258,12 @@ def my_checkpoint(step, t, dt, state): logger.info("Fluid solution failed health check.") logger.info(message) # do this on all ranks - if do_viz or errors > 0: + if do_viz or errored: from mirgecom.simutil import sim_visualization sim_visualization(discr, io_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) - if errors > 0: + if errored: a = 1/0 print(f"{a=}") From 080513e5c324bf88a0d4aa09c0fc4bc7ae29bfaf Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 11 Jun 2021 15:43:35 -0500 Subject: [PATCH 148/385] Udpate error handling per @thomasgibson review. --- examples/autoignition-mpi.py | 3 +-- examples/lump-mpi.py | 25 +++++++++++++------------ examples/mixture-mpi.py | 21 ++++++++++----------- examples/pulse-mpi.py | 18 ++++++++++-------- examples/scalar-lump-mpi.py | 19 +++++++++---------- examples/sod-mpi.py | 17 ++++++++--------- examples/vortex-mpi.py | 17 ++++++++--------- 7 files changed, 59 insertions(+), 61 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 3f6f089a5..bb42bc3fc 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -264,8 +264,7 @@ def my_checkpoint(step, t, dt, state): step=step, t=t, overwrite=True) if errored: - a = 1/0 - print(f"{a=}") + raise RuntimeError("Error detected by user checkpoint, exiting.") return state diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 231b9b2fe..cbc6f2b08 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -65,7 +65,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): dim = 3 nel_1d = 16 order = 3 - exittol = .09 t_final = 0.01 current_cfl = 1.0 vel = np.zeros(shape=(dim,)) @@ -158,30 +157,32 @@ def my_checkpoint(step, t, dt, state): if rank == 0: logger.info(status_msg) - errors = 0 + errored = False if do_health: from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ or check_range_local(discr, "vol", dv.pressure): - errors = 1 + errored = True message = "Invalid pressure data found.\n" - if np.max(component_errors) > exittol: - errors = errors + 1 - message += "Solution errors exceed tolerance.\n" - errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) - if errors > 0: + exittol = .09 + if max(component_errors) > exittol: + errored = True + message += "Solution diverged from exact_mix.\n" + comm = discr.mpi_communicator + if comm is not None: + errored = comm.allreduce(errored, op=MPI.LOR) + if errored: if rank == 0: logger.info("Fluid solution failed health check.") logger.info(message) # do this on all ranks - if do_viz or errors > 0: + if do_viz or errored > 0: from mirgecom.simutil import sim_visualization sim_visualization(discr, io_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) - if errors > 0: - a = 1/0 - print(f"{a=}") + if errored: + raise RuntimeError("Error detected by user checkpoint, exiting.") return state diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index a3480fffc..5d31c7efc 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -178,30 +178,29 @@ def my_checkpoint(step, t, dt, state): if rank == 0: logger.info(status_msg) - errors = 0 + errored = False if do_health: from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ or check_range_local(discr, "vol", dv.pressure): - errors = 1 + errored = True message = "Invalid pressure data found.\n" - if np.max(component_errors) > exittol: - errors = errors + 1 - message += "Solution errors exceed tolerance.\n" - errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) - if errors > 0: + if max(component_errors) > exittol: + errored = True + message += "Solution diverged from exact_mix.\n" + errored = discr.mpi_communicator.allreduce(errored, op=MPI.LOR) + if errored: if rank == 0: logger.info("Fluid solution failed health check.") logger.info(message) # do this on all ranks - if do_viz or errors > 0: + if do_viz or errored: from mirgecom.simutil import sim_visualization sim_visualization(discr, io_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) - if errors > 0: - a = 1/0 - print(f"{a=}") + if errored: + raise RuntimeError("Error detected by user checkpoint, exiting.") return state diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 49c82163a..bca955766 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -165,32 +165,34 @@ def my_checkpoint(step, t, dt, state): if do_status: # This is bad, logging already completely replaces this from mirgecom.io import make_status_message dv = eos.dependent_vars(state) + # this is also bad - no option for user customization, field selection status_msg = make_status_message(discr=discr, t=t, step=step, dt=dt, cfl=current_cfl, dependent_vars=dv) if rank == 0: logger.info(status_msg) - errors = 0 + errored = False if do_health: from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", pressure) \ or check_range_local(discr, "vol", pressure): - errors = 1 + errored = True message = "Invalid pressure data found.\n" - errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) - if errors > 0: + comm = discr.mpi_communicator + if comm is not None: + errored = comm.allreduce(errored, op=MPI.LOR) + if errored: if rank == 0: logger.info("Fluid solution failed health check.") logger.info(message) # do this on all ranks - if do_viz or errors > 0: + if do_viz or errored: from mirgecom.simutil import sim_visualization sim_visualization(discr, io_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) - if errors > 0: - a = 1/0 - print(f"{a=}") + if errored: + raise RuntimeError("Error detected by user checkpoint, exiting.") return state diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 0bc82a453..07dcab149 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -168,30 +168,29 @@ def my_checkpoint(step, t, dt, state): if rank == 0: logger.info(status_msg) - errors = 0 + errored = False if do_health: from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ or check_range_local(discr, "vol", dv.pressure): - errors = 1 + errored = True message = "Invalid pressure data found.\n" - if np.max(component_errors) > exittol: - errors = errors + 1 + if max(component_errors) > exittol: + errored = True message += "Solution errors exceed tolerance.\n" - errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) - if errors > 0: + errored = discr.mpi_communicator.allreduce(errored, op=MPI.LOR) + if errored: if rank == 0: logger.info("Fluid solution failed health check.") logger.info(message) # do this on all ranks - if do_viz or errors > 0: + if do_viz or errored: from mirgecom.simutil import sim_visualization sim_visualization(discr, io_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) - if errors > 0: - a = 1/0 - print(f"{a=}") + if errored: + raise RuntimeError("Error detected by user checkpoint, exiting.") return state diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 205afaeeb..f2dd8763d 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -158,30 +158,29 @@ def my_checkpoint(step, t, dt, state): if rank == 0: logger.info(status_msg) - errors = 0 + errored = False if do_health: from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ or check_range_local(discr, "vol", dv.pressure): - errors = 1 + errored = True message = "Invalid pressure data found.\n" if np.max(component_errors) > exittol: - errors = errors + 1 + errored = True message += "Solution errors exceed tolerance.\n" - errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) - if errors > 0: + errored = discr.mpi_communicator.allreduce(errored, op=MPI.LOR) + if errored: if rank == 0: logger.info("Fluid solution failed health check.") logger.info(message) # do this on all ranks - if do_viz or errors > 0: + if do_viz or errored: from mirgecom.simutil import sim_visualization sim_visualization(discr, io_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) - if errors > 0: - a = 1/0 - print(f"{a=}") + if errored: + raise RuntimeError("Error detected by user checkpoint, exiting.") return state diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 4438bbf52..8cae0411e 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -201,30 +201,29 @@ def my_checkpoint(step, t, dt, state): if rank == 0: logger.info(status_msg) - errors = 0 + errored = False if do_health: from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ or check_range_local(discr, "vol", dv.pressure): - errors = 1 + errored = True message = "Invalid pressure data found.\n" if np.max(component_errors) > exittol: - errors = errors + 1 + errored = True message += "Solution errors exceed tolerance.\n" - errors = discr.mpi_communicator.allreduce(errors, op=MPI.SUM) - if errors > 0: + errored = discr.mpi_communicator.allreduce(errored, op=MPI.LOR) + if errored: if rank == 0: logger.info("Fluid solution failed health check.") logger.info(message) # do this on all ranks - if do_viz or errors > 0: + if do_viz or errored: from mirgecom.simutil import sim_visualization sim_visualization(discr, io_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) - if errors > 0: - a = 1/0 - print(f"{a=}") + if errored: + raise RuntimeError("Error detected by user checkpoint, exiting.") return state From 355a2516a2e587a3cb9652564ed0987e24a425a4 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 11 Jun 2021 17:04:52 -0500 Subject: [PATCH 149/385] Update API per updated grudge --- mirgecom/inviscid.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 08f963fcb..bbcd85726 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -86,7 +86,8 @@ def get_inviscid_timestep(discr, eos, cv): from grudge.dt_utils import characteristic_lengthscales from mirgecom.fluid import compute_wavespeed return ( - characteristic_lengthscales(discr)/compute_wavespeed(discr, eos, cv) + characteristic_lengthscales(cv.array_context, discr) + / compute_wavespeed(discr, eos, cv) ) From df41a5ded2c33a2b3ead04b28b5df06eca24ac1c Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 11 Jun 2021 17:05:27 -0500 Subject: [PATCH 150/385] Switch back to grudge proper. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8a73fd734..5a5d4b277 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ psutil --editable git+https://github.com/inducer/modepy.git#egg=modepy --editable git+https://github.com/inducer/arraycontext.git#egg=arraycontext --editable git+https://github.com/inducer/meshmode.git#egg=meshmode ---editable git+https://github.com/inducer/grudge.git@nongeo-factors-per-group#egg=grudge +--editable git+https://github.com/inducer/grudge.git#egg=grudge --editable git+https://github.com/inducer/pytato.git#egg=pytato --editable git+https://github.com/ecisneros8/pyrometheus.git#egg=pyrometheus --editable git+https://github.com/illinois-ceesd/logpyle.git#egg=logpyle From 382462799abde2b21eb993c20f57781791a01e7c Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 11 Jun 2021 17:25:05 -0500 Subject: [PATCH 151/385] Update timestep API per CV array container --- mirgecom/simutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 8eb55bc58..16be6749d 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -73,7 +73,7 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, from grudge.op import nodal_min mydt = cfl * nodal_min( discr, "vol", - get_inviscid_timestep(discr=discr, eos=eos, q=state) + get_inviscid_timestep(discr=discr, eos=eos, cv=state) ) return min(mydt, dt_left) From 70ff6dc6302db3a8a5d1251b8fcbfd0806b0ea0e Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 11 Jun 2021 20:24:57 -0500 Subject: [PATCH 152/385] Sharpen the handling of post-stepping (exceptional) io per @majosm. --- examples/autoignition-mpi.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index bb42bc3fc..0ad089b17 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -40,7 +40,8 @@ from mirgecom.simutil import ( inviscid_sim_timestep, check_step, - generate_and_distribute_mesh + generate_and_distribute_mesh, + sim_visualization ) from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -221,20 +222,15 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) + eos.get_species_source_terms(state)) - def my_checkpoint(step, t, dt, state): + def my_checkpoint(step, t, dt, state, force=False): from mirgecom.simutil import check_step - do_status = check_step(step=step, interval=nstatus) - do_viz = check_step(step=step, interval=nviz) - do_health = check_step(step=step, interval=nhealth) + do_status = force or check_step(step=step, interval=nstatus) + do_viz = force or check_step(step=step, interval=nviz) + do_health = force or check_step(step=step, interval=nhealth) if do_status or do_viz or do_health: dv = eos.dependent_vars(state) reaction_rates = eos.get_production_rates(state) - io_fields = [ - ("cv", state), - ("dv", dv), - ("reaction_rates", reaction_rates) - ] if do_status: # This is bad, logging already completely replaces this from mirgecom.io import make_status_message @@ -245,21 +241,27 @@ def my_checkpoint(step, t, dt, state): errored = False if do_health: + health_message = "" from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ or check_range_local(discr, "vol", dv.pressure): errored = True - message = "Invalid pressure data found.\n" + health_message += "Invalid pressure data found.\n" comm = discr.mpi_communicator if comm is not None: errored = comm.allreduce(errored, op=MPI.LOR) if errored: if rank == 0: logger.info("Fluid solution failed health check.") - logger.info(message) # do this on all ranks + logger.info(health_message) # do this on all ranks to capture all if do_viz or errored: - from mirgecom.simutil import sim_visualization + reaction_rates = eos.get_production_rates(state) + io_fields = [ + ("cv", state), + ("dv", dv), + ("reaction_rates", reaction_rates) + ] sim_visualization(discr, io_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) @@ -275,11 +277,11 @@ def my_checkpoint(step, t, dt, state): t=current_t, t_final=t_final, eos=eos, dim=dim) if not check_step(current_step, nviz): # If final step not an output step - if rank == 0: + if rank == 0: # Then likely something went wrong logger.info("Checkpointing final state ...") my_checkpoint(current_step, t=current_t, dt=(current_t - checkpoint_t), - state=current_state) + state=current_state, force=True) if __name__ == "__main__": From cccff7f79efe151cdd6552c9d71991296443bd73 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 12 Jun 2021 12:38:59 -0500 Subject: [PATCH 153/385] Sharpen the valid ranges specific to each example, and remove default values in range check --- examples/autoignition-mpi.py | 8 ++++++-- examples/lump-mpi.py | 2 +- examples/mixture-mpi.py | 2 +- examples/pulse-mpi.py | 3 ++- examples/scalar-lump-mpi.py | 3 ++- examples/sod-mpi.py | 2 +- examples/vortex-mpi.py | 8 ++++---- mirgecom/simutil.py | 2 +- test/test_simutil.py | 11 +++++++---- 9 files changed, 25 insertions(+), 16 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 0ad089b17..78155904c 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -244,16 +244,20 @@ def my_checkpoint(step, t, dt, state, force=False): health_message = "" from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure): + or check_range_local(discr, "vol", dv.pressure, 1e5, 2.4e5): errored = True health_message += "Invalid pressure data found.\n" + if check_range_local(discr, "vol", dv.temperature, 1.4e3, 3.3e3): + errored = True + health_message += "Temperature data exceeded healthy range.\n" comm = discr.mpi_communicator if comm is not None: errored = comm.allreduce(errored, op=MPI.LOR) if errored: if rank == 0: logger.info("Fluid solution failed health check.") - logger.info(health_message) # do this on all ranks to capture all + if health_message: # capture any rank's health message + logger.info(f"{rank=}:{health_message}") if do_viz or errored: reaction_rates = eos.get_production_rates(state) diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index cbc6f2b08..9d5a3cd76 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -161,7 +161,7 @@ def my_checkpoint(step, t, dt, state): if do_health: from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure): + or check_range_local(discr, "vol", dv.pressure, .9, 1.1): errored = True message = "Invalid pressure data found.\n" exittol = .09 diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 5d31c7efc..828adfe6e 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -182,7 +182,7 @@ def my_checkpoint(step, t, dt, state): if do_health: from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure): + or check_range_local(discr, "vol", dv.pressure, 1e5, 1.1e5): errored = True message = "Invalid pressure data found.\n" if max(component_errors) > exittol: diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index bca955766..97a6921dc 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -175,7 +175,8 @@ def my_checkpoint(step, t, dt, state): if do_health: from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", pressure) \ - or check_range_local(discr, "vol", pressure): + or check_range_local(discr, "vol", pressure, min_value=.8, + max_value=1.5): errored = True message = "Invalid pressure data found.\n" comm = discr.mpi_communicator diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 07dcab149..7b053dbaa 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -172,7 +172,8 @@ def my_checkpoint(step, t, dt, state): if do_health: from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure): + or check_range_local(discr, "vol", dv.pressure, min_value=.9, + max_value=1.1): errored = True message = "Invalid pressure data found.\n" if max(component_errors) > exittol: diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index f2dd8763d..fbf62e87f 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -162,7 +162,7 @@ def my_checkpoint(step, t, dt, state): if do_health: from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure): + or check_range_local(discr, "vol", dv.pressure, .09, 1.1): errored = True message = "Invalid pressure data found.\n" if np.max(component_errors) > exittol: diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 8cae0411e..6a7536ca8 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -139,8 +139,8 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal logmgr_add_many_discretization_quantities(logmgr, discr, dim, extract_vars_for_logging, units_for_logging) - logmgr.add_watches(["step.max", "t_step.max", "t_log.max", - "min_temperature", "L2_norm_momentum1"]) + logmgr.add_watches(["step.max", "t_step.max", + "min_pressure", "max_pressure"]) try: logmgr.add_watches(["memory_usage_python.max", "memory_usage_gpu.max"]) @@ -205,7 +205,7 @@ def my_checkpoint(step, t, dt, state): if do_health: from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure): + or check_range_local(discr, "vol", dv.pressure, .2, 1.02): errored = True message = "Invalid pressure data found.\n" if np.max(component_errors) > exittol: @@ -229,7 +229,7 @@ def my_checkpoint(step, t, dt, state): current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, + pre_step_callback=my_checkpoint, logmgr=logmgr, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 6df9088c3..cc0ed2594 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -125,7 +125,7 @@ def sim_visualization(discr, io_fields, visualizer, vizname, ) -def check_range_local(discr, dd, field, min_value=0, max_value=np.inf): +def check_range_local(discr, dd, field, min_value, max_value): """Check for any negative values.""" return ( op.nodal_min_loc(discr, dd, field) < min_value diff --git a/test/test_simutil.py b/test/test_simutil.py index 572504bf2..63b8990e3 100644 --- a/test/test_simutil.py +++ b/test/test_simutil.py @@ -68,8 +68,9 @@ def test_basic_cfd_healthcheck(actx_factory): pressure = eos.pressure(cv) from mirgecom.simutil import check_range_local - assert check_range_local(discr, "vol", mass) - assert check_range_local(discr, "vol", pressure, min_value=1e-6) + assert check_range_local(discr, "vol", mass, min_value=0, max_value=np.inf) + assert check_range_local(discr, "vol", pressure, min_value=1e-6, + max_value=np.inf) # Let's make another very bad state (nans) mass = 1*ones @@ -99,8 +100,10 @@ def test_basic_cfd_healthcheck(actx_factory): cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) pressure = eos.pressure(cv) - assert not check_naninf_local(discr, "vol", pressure) - assert not check_range_local(discr, "vol", pressure) + assert not check_naninf_local(discr, "vol", pressure, min_value=0, + max_value=np.inf) + assert not check_range_local(discr, "vol", pressure, min_value=0, + max_value=np.inf) def test_analytic_comparison(actx_factory): From 1772b86732ce9d5ad9a5b349b09b41e5e33747d0 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 12 Jun 2021 13:33:04 -0500 Subject: [PATCH 154/385] Correct call to naninf in test. --- test/test_simutil.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test_simutil.py b/test/test_simutil.py index 63b8990e3..9f78062db 100644 --- a/test/test_simutil.py +++ b/test/test_simutil.py @@ -100,8 +100,7 @@ def test_basic_cfd_healthcheck(actx_factory): cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) pressure = eos.pressure(cv) - assert not check_naninf_local(discr, "vol", pressure, min_value=0, - max_value=np.inf) + assert not check_naninf_local(discr, "vol", pressure) assert not check_range_local(discr, "vol", pressure, min_value=0, max_value=np.inf) From c3518686a3faf3a3ce37c9c4d0975c963f449a54 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 12 Jun 2021 15:22:07 -0500 Subject: [PATCH 155/385] Tweak into place after merge with main. --- mirgecom/boundary.py | 1 - mirgecom/euler.py | 28 +--------------------------- mirgecom/flux.py | 1 - mirgecom/inviscid.py | 7 +++---- mirgecom/simutil.py | 2 +- test/test_init.py | 2 +- 6 files changed, 6 insertions(+), 35 deletions(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 2abdc13d1..86563fd7c 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -306,7 +306,6 @@ def __init__(self, userfunc): least one parameter that specifies the coordinates at which to prescribe the solution. """ - from warnings import warn warn("Do not use PrescribedBoundary; use PrescribedInvscidBoundary. This" "boundary type will disappear soon.", DeprecationWarning, stacklevel=2) diff --git a/mirgecom/euler.py b/mirgecom/euler.py index 147fb5e13..548b70a8b 100644 --- a/mirgecom/euler.py +++ b/mirgecom/euler.py @@ -114,31 +114,6 @@ def euler_operator(discr, eos, boundaries, cv, t=0.0): ) q = -dg_div(discr, inviscid_flux_vol.join(), inviscid_flux_bnd.join()) return make_conserved(discr.dim, q=q) -======= - vol_weak = discr.weak_div(inviscid_flux(discr=discr, eos=eos, cv=cv).join()) - - boundary_flux = ( - _facial_flux(discr=discr, eos=eos, cv_tpair=interior_trace_pair(discr, cv)) - + sum( - _facial_flux( - discr, eos=eos, - cv_tpair=TracePair( - part_pair.dd, - interior=split_conserved(discr.dim, part_pair.int), - exterior=split_conserved(discr.dim, part_pair.ext))) - for part_pair in cross_rank_trace_pairs(discr, cv.join())) - + sum( - _facial_flux( - discr=discr, eos=eos, - cv_tpair=boundaries[btag].boundary_pair( - discr, eos=eos, btag=btag, t=t, cv=cv) - ) - for btag in boundaries) - ).join() - - return split_conserved( - discr.dim, discr.inverse_mass(vol_weak - discr.face_mass(boundary_flux)) - ) def inviscid_operator(discr, eos, boundaries, q, t=0.0): @@ -146,8 +121,7 @@ def inviscid_operator(discr, eos, boundaries, q, t=0.0): from warnings import warn warn("Do not call inviscid_operator; it is now called euler_operator. This" "function will disappear August 1, 2021", DeprecationWarning, stacklevel=2) - return euler_operator(discr, eos, boundaries, split_conserved(discr.dim, q), t) ->>>>>>> main + return euler_operator(discr, eos, boundaries, make_conserved(discr.dim, q=q), t) # By default, run unitless diff --git a/mirgecom/flux.py b/mirgecom/flux.py index b736ae062..c6af9551f 100644 --- a/mirgecom/flux.py +++ b/mirgecom/flux.py @@ -122,7 +122,6 @@ def central_vector_flux(trace_pair, normal): def lfr_flux(cv_tpair, f_tpair, normal, lam): - r"""Compute Lax-Friedrichs/Rusanov flux after [Hesthaven_2008]_, Section 6.6. The Lax-Friedrichs/Rusanov flux is calculated as: diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 3033d054b..81e3bd2d7 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -121,19 +121,18 @@ def get_inviscid_timestep(discr, eos, cv): eos: mirgecom.eos.GasEOS Implementing the pressure and temperature functions for returning pressure and temperature as a function of the state q. - cv: :class:`ConservedVars` + cv: :class:`~mirgecom.fluid.ConservedVars` Fluid solution Returns ------- class:`~meshmode.dof_array.DOFArray` The maximum stable timestep at each node. """ - from grudge.dt_utils import (dt_non_geometric_factor, - dt_geometric_factors) + from grudge.dt_utils import characteristic_length_scales from mirgecom.fluid import compute_wavespeed return ( - dt_non_geometric_factor(discr) * dt_geometric_factors(discr) + characteristic_length_scales(cv.array_context, discr) / compute_wavespeed(eos, cv) ) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index ad2941f5a..bd3fa9aa5 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -141,7 +141,7 @@ def sim_checkpoint(discr, visualizer, eos, cv, vizname, exact_soln=None, expected_state = exact_soln(x_vec=nodes, eos=eos, time=t) exp_resid = cv - expected_state err_norms = [discr.norm(v, np.inf) for v in exp_resid.join()] - maxerr = discr.norm(ex_resid.join(), np.inf()) + maxerr = discr.norm(exp_resid.join(), np.inf()) if do_viz: io_fields = [ diff --git a/test/test_init.py b/test/test_init.py index ed883fe75..18f03152b 100644 --- a/test/test_init.py +++ b/test/test_init.py @@ -259,7 +259,7 @@ def test_uniform(ctx_factory, dim): assert discr.norm(initsoln.mass - 1.0, np.inf) < tol assert discr.norm(initsoln.energy - 2.5, np.inf) < tol - print(f"Uniform Soln:{cv}") + print(f"Uniform Soln:{initsoln}") eos = IdealSingleGas() p = eos.pressure(initsoln) print(f"Press:{p}") From c93c20d6f8d2f16857c6290216cf72ceb5ec57bf Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 12 Jun 2021 15:29:03 -0500 Subject: [PATCH 156/385] Tweak into place after merging disc-init --- examples/doublemach-mpi.py | 1 - mirgecom/inviscid.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 33ee4ccaa..e093054f8 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -37,7 +37,6 @@ from grudge.shortcuts import make_visualizer -from mirgecom.fluid import make_conserved from mirgecom.navierstokes import ns_operator from mirgecom.artificial_viscosity import ( av_operator, diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 1996f4879..91dc66f18 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -135,6 +135,7 @@ def get_inviscid_timestep(discr, eos, cv): / compute_wavespeed(eos, cv) ) + def get_inviscid_cfl(discr, eos, dt, cv): """Calculate and return node-local CFL based on current state and timestep. From ed16c58f57d4806447541c893010ddbe20bc7d19 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 12 Jun 2021 15:49:11 -0500 Subject: [PATCH 157/385] Tweak into place after big new merge --- mirgecom/inviscid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 91dc66f18..9164c902c 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -128,10 +128,10 @@ def get_inviscid_timestep(discr, eos, cv): class:`~meshmode.dof_array.DOFArray` The maximum stable timestep at each node. """ - from grudge.dt_utils import characteristic_length_scales + from grudge.dt_utils import characteristic_lengthscales from mirgecom.fluid import compute_wavespeed return ( - characteristic_length_scales(cv.array_context, discr) + characteristic_lengthscales(cv.array_context, discr) / compute_wavespeed(eos, cv) ) From a9b799037b97e132b356da5a940de400f370087d Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 12 Jun 2021 18:57:22 -0500 Subject: [PATCH 158/385] Use verbs for function names. --- mirgecom/simutil.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index cc0ed2594..d88bae91e 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -5,7 +5,7 @@ .. autofunction:: check_step .. autofunction:: inviscid_sim_timestep -.. autofunction:: sim_visualization +.. autofunction:: write_visfile Diagnostic utilities -------------------- @@ -86,11 +86,9 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, return mydt -def sim_visualization(discr, io_fields, visualizer, vizname, - step=0, t=0, overwrite=False, vis_timer=None): - """Visualize the simulation fields. - - Write VTK output for the specified fields. +def write_visfile(discr, io_fields, visualizer, vizname, + step=0, t=0, overwrite=False, vis_timer=None): + """Write VTK output for the fields specified in *io_fields*. Parameters ---------- From 100df7741cd1d89a9b6bb26057918623137b9ce6 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 12 Jun 2021 21:42:39 -0500 Subject: [PATCH 159/385] Use new name for visfile util, switch autoignition and vortex over to not use built-in status message --- examples/autoignition-mpi.py | 90 ++++++++++++++++++++++++++++-------- examples/lump-mpi.py | 6 +-- examples/mixture-mpi.py | 4 +- examples/pulse-mpi.py | 6 +-- examples/scalar-lump-mpi.py | 6 +-- examples/sod-mpi.py | 6 +-- examples/vortex-mpi.py | 43 +++++++++++------ mirgecom/steppers.py | 12 +---- 8 files changed, 116 insertions(+), 57 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 78155904c..31d82ffe4 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -30,18 +30,23 @@ from functools import partial from meshmode.array_context import PyOpenCLArrayContext + from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.eager import EagerDGDiscretization from grudge.shortcuts import make_visualizer +from logpyle import IntervalTimer, set_dt +from mirgecom.euler import extract_vars_for_logging, units_for_logging +from mirgecom.profiling import PyOpenCLProfilingArrayContext + from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, check_step, generate_and_distribute_mesh, - sim_visualization + write_visfile ) from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -52,6 +57,14 @@ from mirgecom.initializers import MixtureInitializer from mirgecom.eos import PyrometheusMixture +from mirgecom.logging_quantities import ( + initialize_logmgr, + logmgr_add_many_discretization_quantities, + logmgr_add_device_name, + logmgr_add_device_memory_usage, + set_sim_state +) + import cantera import pyrometheus as pyro @@ -59,11 +72,30 @@ @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False): +def main(ctx_factory=cl.create_some_context, use_logmgr=False, + use_leap=False, use_profiling=False, casename=None): """Drive example.""" cl_ctx = ctx_factory() - queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, + + if casename is None: + casename = "mirgecom" + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + + logmgr = initialize_logmgr(use_logmgr, + filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) + + if use_profiling: + queue = cl.CommandQueue(cl_ctx, + properties=cl.command_queue_properties.PROFILING_ENABLE) + actx = PyOpenCLProfilingArrayContext(queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), + logmgr=logmgr) + else: + queue = cl.CommandQueue(cl_ctx) + actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) dim = 2 @@ -94,10 +126,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): box_ur = 0.005 debug = False - from mpi4py import MPI - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - from meshmode.mesh.generation import generate_regular_rect_mesh generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,) * dim, b=(box_ur,) * dim, nelements_per_axis=(nel_1d,) * dim) @@ -109,6 +137,19 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): ) nodes = thaw(actx, discr.nodes()) + if logmgr: + logmgr_add_device_name(logmgr, queue) + logmgr_add_device_memory_usage(logmgr, queue) + logmgr_add_many_discretization_quantities(logmgr, discr, dim, + extract_vars_for_logging, units_for_logging) + + logmgr.add_watches(["step.max", "t_step.max", + "min_pressure", "max_pressure", + "min_temperature", "max_temperature"]) + + vis_timer = IntervalTimer("t_vis", "Time spent visualizing") + logmgr.add_quantity(vis_timer) + # {{{ Set up initial state using Cantera # Use Cantera for initialization @@ -160,7 +201,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): # Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and # generates a set of methods to calculate chemothermomechanical properties and # states for this particular mechanism. - casename = "autoignition" pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np) eos = PyrometheusMixture(pyrometheus_mechanism, temperature_guess=init_temperature) @@ -222,22 +262,29 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) + eos.get_species_source_terms(state)) + def post_step_stuff(step, t, dt, state): + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state + def my_checkpoint(step, t, dt, state, force=False): from mirgecom.simutil import check_step do_status = force or check_step(step=step, interval=nstatus) do_viz = force or check_step(step=step, interval=nviz) do_health = force or check_step(step=step, interval=nhealth) + if logmgr: + logmgr.tick_before() + if do_status or do_viz or do_health: dv = eos.dependent_vars(state) reaction_rates = eos.get_production_rates(state) - if do_status: # This is bad, logging already completely replaces this - from mirgecom.io import make_status_message - status_msg = make_status_message(discr=discr, t=t, step=step, dt=dt, - cfl=current_cfl, dependent_vars=dv) + if do_status: # wish to control watch quantities output, too if rank == 0: - logger.info(status_msg) + logger.info(f"time={t}: {dt=},{current_cfl=}\n") errored = False if do_health: @@ -266,8 +313,8 @@ def my_checkpoint(step, t, dt, state, force=False): ("dv", dv), ("reaction_rates", reaction_rates) ] - sim_visualization(discr, io_fields, visualizer, vizname=casename, - step=step, t=t, overwrite=True) + write_visfile(discr, io_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) if errored: raise RuntimeError("Error detected by user checkpoint, exiting.") @@ -277,6 +324,7 @@ def my_checkpoint(step, t, dt, state, force=False): current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_checkpoint, + post_step_callback=post_step_stuff, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) @@ -289,7 +337,13 @@ def my_checkpoint(step, t, dt, state, force=False): if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main(use_leap=False) + logging.basicConfig(format="%(message)s", level=logging.INFO) + use_profiling = True + use_logging = True + use_leap = False + casename = "autoignition" + + main(use_profiling=use_profiling, use_logmgr=use_logging, use_leap=use_leap, + casename=casename) # vim: foldmethod=marker diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 9d5a3cd76..8ca64e5d2 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -177,9 +177,9 @@ def my_checkpoint(step, t, dt, state): logger.info(message) # do this on all ranks if do_viz or errored > 0: - from mirgecom.simutil import sim_visualization - sim_visualization(discr, io_fields, visualizer, vizname=casename, - step=step, t=t, overwrite=True) + from mirgecom.simutil import write_visfile + write_visfile(discr, io_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) if errored: raise RuntimeError("Error detected by user checkpoint, exiting.") diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 828adfe6e..ca20bce37 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -195,8 +195,8 @@ def my_checkpoint(step, t, dt, state): logger.info(message) # do this on all ranks if do_viz or errored: - from mirgecom.simutil import sim_visualization - sim_visualization(discr, io_fields, visualizer, vizname=casename, + from mirgecom.simutil import write_visfile + write_visfile(discr, io_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) if errored: diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 97a6921dc..b10aafa7a 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -188,9 +188,9 @@ def my_checkpoint(step, t, dt, state): logger.info(message) # do this on all ranks if do_viz or errored: - from mirgecom.simutil import sim_visualization - sim_visualization(discr, io_fields, visualizer, vizname=casename, - step=step, t=t, overwrite=True) + from mirgecom.simutil import write_visfile + write_visfile(discr, io_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) if errored: raise RuntimeError("Error detected by user checkpoint, exiting.") diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 7b053dbaa..a434835b1 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -186,9 +186,9 @@ def my_checkpoint(step, t, dt, state): logger.info(message) # do this on all ranks if do_viz or errored: - from mirgecom.simutil import sim_visualization - sim_visualization(discr, io_fields, visualizer, vizname=casename, - step=step, t=t, overwrite=True) + from mirgecom.simutil import write_visfile + write_visfile(discr, io_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) if errored: raise RuntimeError("Error detected by user checkpoint, exiting.") diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index fbf62e87f..4e4526853 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -175,9 +175,9 @@ def my_checkpoint(step, t, dt, state): logger.info(message) # do this on all ranks if do_viz or errored: - from mirgecom.simutil import sim_visualization - sim_visualization(discr, io_fields, visualizer, vizname=casename, - step=step, t=t, overwrite=True) + from mirgecom.simutil import write_visfile + write_visfile(discr, io_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) if errored: raise RuntimeError("Error detected by user checkpoint, exiting.") diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 6a7536ca8..866479cf8 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -51,13 +51,16 @@ from mirgecom.initializers import Vortex2D from mirgecom.eos import IdealSingleGas -from logpyle import IntervalTimer +from logpyle import IntervalTimer, set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging -from mirgecom.logging_quantities import (initialize_logmgr, - logmgr_add_many_discretization_quantities, logmgr_add_device_name, - logmgr_add_device_memory_usage) - +from mirgecom.logging_quantities import ( + initialize_logmgr, + logmgr_add_many_discretization_quantities, + logmgr_add_device_name, + logmgr_add_device_memory_usage, + set_sim_state +) logger = logging.getLogger(__name__) @@ -175,12 +178,22 @@ def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) + def post_step_stuff(step, t, dt, state): + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state + def my_checkpoint(step, t, dt, state): from mirgecom.simutil import check_step do_status = check_step(step=step, interval=nstatus) do_viz = check_step(step=step, interval=nviz) do_health = check_step(step=step, interval=nhealth) + if logmgr: + logmgr.tick_before() + if do_status or do_viz or do_health: from mirgecom.simutil import compare_fluid_solutions dv = eos.dependent_vars(state) @@ -195,11 +208,12 @@ def my_checkpoint(step, t, dt, state): ] if do_status: - status_msg = ( - "\n------- errors=" - + ", ".join("%.3g" % en for en in component_errors)) if rank == 0: - logger.info(status_msg) + logger.info( + f"time={t}: \n" + "---- errors=" + ",".join("%.3g" % en for en + in component_errors) + ) errored = False if do_health: @@ -218,9 +232,9 @@ def my_checkpoint(step, t, dt, state): logger.info(message) # do this on all ranks if do_viz or errored: - from mirgecom.simutil import sim_visualization - sim_visualization(discr, io_fields, visualizer, vizname=casename, - step=step, t=t, overwrite=True) + from mirgecom.simutil import write_visfile + write_visfile(discr, io_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) if errored: raise RuntimeError("Error detected by user checkpoint, exiting.") @@ -229,7 +243,8 @@ def my_checkpoint(step, t, dt, state): current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, logmgr=logmgr, + pre_step_callback=my_checkpoint, + post_step_callback=post_step_stuff, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) @@ -247,7 +262,7 @@ def my_checkpoint(step, t, dt, state): if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) - use_profiling = True + use_profiling = False use_logging = True use_leap = False diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index b069f38e5..3a4413edc 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -27,7 +27,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ - from logpyle import set_dt from mirgecom.logging_quantities import set_sim_state @@ -98,12 +97,11 @@ def _advance_state_stepper_func(rhs, timestepper, get_timestep, state = timestepper(state=state, t=t, dt=dt, rhs=rhs) t += dt + istep += 1 if post_step_callback is not None: state = post_step_callback(state=state, step=istep, t=t, dt=dt) - istep += 1 - if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) @@ -170,9 +168,6 @@ def _advance_state_leap(rhs, timestepper, get_timestep, rhs, t, dt, state) while t < t_final: - if logmgr: - logmgr.tick_before() - dt = get_timestep(state=state) if dt < 0: return istep, t, state @@ -195,11 +190,6 @@ def _advance_state_leap(rhs, timestepper, get_timestep, istep += 1 - if logmgr: - set_dt(logmgr, dt) - set_sim_state(logmgr, dim, state, eos) - logmgr.tick_after() - return istep, t, state From 6f7ef047595d99e161d4b32ebd5473a278964efa Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sun, 13 Jun 2021 04:46:09 -0500 Subject: [PATCH 160/385] Clean up logging a bit, attempt to use nlog to control logging interval. --- examples/autoignition-mpi.py | 28 +++++++++++++--------- examples/vortex-mpi.py | 46 +++++++++++++++++++++--------------- 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 31d82ffe4..7a25ab08c 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -114,6 +114,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, nstatus = 1 nviz = 5 nhealth = 1 + nlog = 1 rank = 0 checkpoint_t = current_t current_step = 0 @@ -143,9 +144,16 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, logmgr_add_many_discretization_quantities(logmgr, discr, dim, extract_vars_for_logging, units_for_logging) - logmgr.add_watches(["step.max", "t_step.max", - "min_pressure", "max_pressure", - "min_temperature", "max_temperature"]) + logmgr.add_watches([ + ("step.max", "step = {value}, "), + ("t_sim.max", "sim time: {value:1.6e} s\n"), + ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), + ("max_pressure", "{value:1.9e})\n"), + ("min_temperature", "------- T (min, max) (K) = ({value:7g}, "), + ("max_temperature", "{value:7g})\n"), + ("t_step.max", "------- step walltime: {value:6g} s, "), + ("t_log.max", "log walltime: {value:6g} s") + ]) vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) @@ -263,7 +271,9 @@ def my_rhs(t, state): + eos.get_species_source_terms(state)) def post_step_stuff(step, t, dt, state): - if logmgr: + do_logend = check_step(step=(step-1), interval=nlog) + + if do_logend and logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) logmgr.tick_after() @@ -271,21 +281,17 @@ def post_step_stuff(step, t, dt, state): def my_checkpoint(step, t, dt, state, force=False): from mirgecom.simutil import check_step - do_status = force or check_step(step=step, interval=nstatus) do_viz = force or check_step(step=step, interval=nviz) do_health = force or check_step(step=step, interval=nhealth) + do_logstart = force or check_step(step=step, interval=nlog) - if logmgr: + if do_logstart and logmgr: logmgr.tick_before() - if do_status or do_viz or do_health: + if do_viz or do_health: dv = eos.dependent_vars(state) reaction_rates = eos.get_production_rates(state) - if do_status: # wish to control watch quantities output, too - if rank == 0: - logger.info(f"time={t}: {dt=},{current_cfl=}\n") - errored = False if do_health: health_message = "" diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 866479cf8..04f5603f8 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -40,7 +40,8 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, - generate_and_distribute_mesh + generate_and_distribute_mesh, + check_step ) from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -106,6 +107,7 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal nstatus = 10 nviz = 10 nhealth = 10 + nlog = 10 rank = 0 checkpoint_t = current_t current_step = 0 @@ -142,8 +144,14 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal logmgr_add_many_discretization_quantities(logmgr, discr, dim, extract_vars_for_logging, units_for_logging) - logmgr.add_watches(["step.max", "t_step.max", - "min_pressure", "max_pressure"]) + logmgr.add_watches([ + ("step.max", "step = {value}, "), + ("t_sim.max", "sim time: {value:1.6e} s\n"), + ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), + ("max_pressure", "{value:1.9e})\n"), + ("t_step.max", "------- step walltime: {value:6g} s, "), + ("t_log.max", "log walltime: {value:6g} s") + ]) try: logmgr.add_watches(["memory_usage_python.max", "memory_usage_gpu.max"]) @@ -179,47 +187,40 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def post_step_stuff(step, t, dt, state): - if logmgr: + do_log = check_step(step=(step-1), interval=nlog) + if do_log and logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) logmgr.tick_after() return state def my_checkpoint(step, t, dt, state): - from mirgecom.simutil import check_step do_status = check_step(step=step, interval=nstatus) do_viz = check_step(step=step, interval=nviz) do_health = check_step(step=step, interval=nhealth) + do_log = check_step(step=step, interval=nlog) - if logmgr: + if do_log and logmgr: logmgr.tick_before() if do_status or do_viz or do_health: from mirgecom.simutil import compare_fluid_solutions - dv = eos.dependent_vars(state) + pressure = eos.pressure(state) vortex_exact = initializer(x_vec=nodes, eos=eos, t=t) component_errors = compare_fluid_solutions(discr, state, vortex_exact) - resid = state - vortex_exact - io_fields = [ - ("cv", state), - ("dv", dv), - ("vortex_exact", vortex_exact), - ("resid", resid) - ] if do_status: if rank == 0: logger.info( - f"time={t}: \n" - "---- errors=" + ",".join("%.3g" % en for en - in component_errors) + "------- errors=" + ",".join("%.3g" % en for en + in component_errors) ) errored = False if do_health: from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure, .2, 1.02): + if check_naninf_local(discr, "vol", pressure) \ + or check_range_local(discr, "vol", pressure, .2, 1.02): errored = True message = "Invalid pressure data found.\n" if np.max(component_errors) > exittol: @@ -232,6 +233,13 @@ def my_checkpoint(step, t, dt, state): logger.info(message) # do this on all ranks if do_viz or errored: + resid = state - vortex_exact + io_fields = [ + ("cv", state), + ("pressure", pressure), + ("vortex_exact", vortex_exact), + ("resid", resid) + ] from mirgecom.simutil import write_visfile write_visfile(discr, io_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) From 5f31dcf5196b1c8e1e49440e25312b9c3199c688 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Sun, 13 Jun 2021 12:53:23 -0500 Subject: [PATCH 161/385] Use better, more descriptive variable names. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andreas Klöckner --- mirgecom/simutil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 16be6749d..5bc0980fc 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -67,8 +67,8 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, """Return the maximum stable dt.""" mydt = dt dt_left = t_final - t - if dt_left < 0: - return 0.0 + if t_remaining < dt: + return max(0, t_remaining) if constant_cfl is True: from grudge.op import nodal_min mydt = cfl * nodal_min( From 270ea3b3023234ca22ae53533c6fcacc81229aba Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Sun, 13 Jun 2021 12:54:55 -0500 Subject: [PATCH 162/385] Use better, more descriptive variable names. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andreas Klöckner --- mirgecom/simutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 5bc0980fc..0dd273427 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -66,7 +66,7 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, t_final, constant_cfl=False): """Return the maximum stable dt.""" mydt = dt - dt_left = t_final - t + t_remaining = t_final - t if t_remaining < dt: return max(0, t_remaining) if constant_cfl is True: From 6ff984d66909a14d628709f83d6decd85a49846b Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Sun, 13 Jun 2021 13:02:18 -0500 Subject: [PATCH 163/385] Update dt calculator to jibe with review suggestions. --- mirgecom/simutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 0dd273427..d0446fa21 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -75,7 +75,7 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, discr, "vol", get_inviscid_timestep(discr=discr, eos=eos, cv=state) ) - return min(mydt, dt_left) + return mydt class ExactSolutionMismatch(Exception): From 2813e264d7225d9e1bfa3a275ab93dfce49cdf8d Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Mon, 14 Jun 2021 06:04:20 -0700 Subject: [PATCH 164/385] Tweak DT finding API for grudge changes --- mirgecom/inviscid.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 7be595dfb..f8957a45d 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -141,15 +141,14 @@ def get_inviscid_timestep(discr, eos, q): class:`~meshmode.dof_array.DOFArray` The maximum stable timestep at each node. """ - from grudge.dt_utils import (dt_non_geometric_factor, - dt_geometric_factors) + from grudge.dt_utils import characteristic_lengthscales from mirgecom.fluid import compute_wavespeed dim = discr.dim cv = split_conserved(dim, q) - + actx = cv.mass.array_context return ( - dt_non_geometric_factor(discr) * dt_geometric_factors(discr) + characteristic_lengthscales(actx, discr) / compute_wavespeed(dim, eos, cv) ) From 68d5551bde6d190b95349a9147d220219a7570c5 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Tue, 15 Jun 2021 05:09:57 -0700 Subject: [PATCH 165/385] Tweak the timstep computation just a tad --- mirgecom/simutil.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index b2d824fff..844778198 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -91,16 +91,14 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, t_final, constant_cfl=False): """Return the maximum stable dt.""" mydt = dt - dt_left = t_final - t - if dt_left < 0: - return 0.0 if constant_cfl is True: from grudge.op import nodal_min mydt = cfl * nodal_min( discr, "vol", get_inviscid_timestep(discr=discr, eos=eos, cv=state) ) - return min(mydt, dt_left) + dt_remaining = max(0, t_final - t) + return min(mydt, dt_remaining) class ExactSolutionMismatch(Exception): From b892481a9935caa07497ac69b85528de67bdeff6 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Tue, 15 Jun 2021 06:22:25 -0700 Subject: [PATCH 166/385] Add simple health utils needed by production drivers. --- mirgecom/simutil.py | 112 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 86 insertions(+), 26 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 844778198..f01adebd6 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1,9 +1,25 @@ """Provide some utilities for building simulation applications. +General utilities +----------------- + .. autofunction:: check_step .. autofunction:: inviscid_sim_timestep .. autoexception:: ExactSolutionMismatch +.. autofunction:: write_visfile .. autofunction:: sim_checkpoint +.. autofunction:: write_restart_file + +Diagnostic utilities +-------------------- + +.. autofunction:: compare_fluid_solutions +.. autofunction:: check_naninf_local +.. autofunction:: check_range_local + +Mesh utilities +-------------- + .. autofunction:: generate_and_distribute_mesh """ @@ -30,40 +46,26 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ - import logging - +import pickle import numpy as np + from meshmode.dof_array import thaw from mirgecom.io import make_status_message from mirgecom.inviscid import get_inviscid_timestep # bad smell? +from meshmode.dof_array import thaw, flatten, unflatten # noqa +from mirgecom.fluid import ConservedVars logger = logging.getLogger(__name__) -class MIRGEComParameters: - """Simple parameters object.""" - - def __init__(self, **kwargs): - """Initialize parameters object.""" - self._parameters = kwargs - - @property - def parameters(self): - """Grab the parameters.""" - return self._parameters - - def update(self, **kwargs): - """Update parameters with new or replacement parameters or values.""" - self._parameters.update(kwargs) - - def read(self, file_path): - """Read new or replacement values from a file at system path *file_path*.""" - import importlib.util - spec = importlib.util.spec_from_file_location("user_parameters", file_path) - foo = importlib.util.module_from_spec(spec) - spec.loader.exec_module(foo) - self._parameters.update(foo.mirgecom_parameters.parameters) +def write_restart_file(actx, restart_dictionary, filename): + from pytools.obj_array import obj_array_vectorize + state = restart_dictionary["state"].join() + restart_dictionary["state"] = obj_array_vectorize(actx.to_numpy, + flatten(state)) + with open(filename, "wb") as f: + pickle.dump(restart_dictionary, f) def check_step(step, interval): @@ -103,7 +105,6 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, class ExactSolutionMismatch(Exception): """Exception class for solution mismatch. - .. attribute:: step .. attribute:: t .. attribute:: state @@ -116,6 +117,43 @@ def __init__(self, step, t, state): self.state = state +def write_visfile(discr, io_fields, visualizer, vizname, + step=0, t=0, overwrite=False, vis_timer=None): + """Write VTK output for the fields specified in *io_fields*. + + Parameters + ---------- + visualizer: + A :class:`meshmode.discretization.visualization.Visualizer` + VTK output object. + io_fields: + List of tuples indicating the (name, data) for each field to write. + """ + from contextlib import nullcontext + from mirgecom.io import make_rank_fname, make_par_fname + + comm = discr.mpi_communicator + rank = 0 + if comm is not None: + rank = comm.Get_rank() + + rank_fn = make_rank_fname(basename=vizname, rank=rank, step=step, t=t) + + if vis_timer: + ctm = vis_timer.start_sub_timer() + else: + ctm = nullcontext() + + with ctm: + visualizer.write_parallel_vtk_file( + comm, rank_fn, io_fields, + overwrite=overwrite, + par_manifest_filename=make_par_fname( + basename=vizname, step=step, t=t + ) + ) + + def sim_checkpoint(discr, visualizer, eos, cv, vizname, exact_soln=None, step=0, t=0, dt=0, cfl=1.0, nstatus=-1, nviz=-1, exittol=1e-16, constant_cfl=False, comm=None, viz_fields=None, overwrite=False, @@ -187,6 +225,28 @@ def sim_checkpoint(discr, visualizer, eos, cv, vizname, exact_soln=None, raise ExactSolutionMismatch(step, t=t, state=cv) +def check_range_local(discr, dd, field, min_value, max_value): + """Check for any negative values.""" + from grudge.op import nodal_min_loc, nodal_max_loc + return ( + nodal_min_loc(discr, dd, field) < min_value + or nodal_max_loc(discr, dd, field) > max_value + ) + + +def check_naninf_local(discr, dd, field): + """Check for any NANs or Infs in the field.""" + from grudge.op import nodal_sum_loc + s = nodal_sum_loc(discr, dd, field) + return np.isnan(s) or (s == np.inf) + + +def compare_fluid_solutions(discr, red_state, blue_state): + """Return inf norm of (*red_state* - *blue_state*) for each component.""" + resid = red_state - blue_state + return [discr.norm(v, np.inf) for v in resid.join()] + + def generate_and_distribute_mesh(comm, generate_mesh): """Generate a mesh and distribute it among all ranks in *comm*. From efbd1f1471579093b216a0b28b331059b88cf4eb Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Tue, 15 Jun 2021 06:51:16 -0700 Subject: [PATCH 167/385] Placate the linters, and remove unneeded util test. --- mirgecom/simutil.py | 4 ++- test/test_util.py | 64 --------------------------------------------- 2 files changed, 3 insertions(+), 65 deletions(-) delete mode 100644 test/test_util.py diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index f01adebd6..9eedb67c7 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -54,12 +54,13 @@ from mirgecom.io import make_status_message from mirgecom.inviscid import get_inviscid_timestep # bad smell? from meshmode.dof_array import thaw, flatten, unflatten # noqa -from mirgecom.fluid import ConservedVars +from mirgecom.fluid import ConservedVars # noqa logger = logging.getLogger(__name__) def write_restart_file(actx, restart_dictionary, filename): + """Pickle the simulation data into a file for use in restarting.""" from pytools.obj_array import obj_array_vectorize state = restart_dictionary["state"].join() restart_dictionary["state"] = obj_array_vectorize(actx.to_numpy, @@ -105,6 +106,7 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, class ExactSolutionMismatch(Exception): """Exception class for solution mismatch. + .. attribute:: step .. attribute:: t .. attribute:: state diff --git a/test/test_util.py b/test/test_util.py deleted file mode 100644 index 822c3cd7e..000000000 --- a/test/test_util.py +++ /dev/null @@ -1,64 +0,0 @@ -__copyright__ = """ -Copyright (C) 2020 University of Illinois Board of Trustees -""" - -__license__ = """ -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -""" - -import logging -from mirgecom.simutil import MIRGEComParameters - -logger = logging.getLogger(__name__) - - -def test_mirgecom_parameters(): - """Quick test of MIRGE-Com parameters container.""" - test_params = MIRGEComParameters(dim=2, order=3, casename="hello") - my_params = test_params.parameters - print(f"{test_params.parameters}") - assert len(my_params) == 3 - assert my_params["dim"] == 2 - assert my_params["order"] == 3 - assert my_params["casename"] == "hello" - - test_params.update(order=4, casename="goodbye", hello="hello") - my_params = test_params.parameters - assert len(my_params) == 4 - assert my_params["order"] == 4 - assert my_params["dim"] == 2 - assert my_params["casename"] == "goodbye" - assert my_params["hello"] == "hello" - - params_string = ( - "from mirgecom.simutil import MIRGEComParameters" - "\nmirgecom_parameters = MIRGEComParameters(" - "\ndim=5, newparam='string')" - ) - - file1 = open("test_params_fjsfjksd.py", "a") - file1.write(params_string) - file1.close() - test_params.read("test_params_fjsfjksd.py") - my_params = test_params.parameters - assert len(my_params) == 5 - assert my_params["dim"] == 5 - assert my_params["newparam"] == "string" - import os - os.remove("test_params_fjsfjksd.py") From d972dd091809724c661a116120b392b5559ae57f Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 15 Jun 2021 19:27:36 -0500 Subject: [PATCH 168/385] Add some restart utils for convenience --- mirgecom/fluid.py | 6 +++++- mirgecom/simutil.py | 18 +++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/mirgecom/fluid.py b/mirgecom/fluid.py index 9fd4e68da..5fadbcd3f 100644 --- a/mirgecom/fluid.py +++ b/mirgecom/fluid.py @@ -321,8 +321,12 @@ def join_conserved(dim, mass, energy, momentum, species_mass=None): def make_conserved(dim, mass=None, energy=None, momentum=None, species_mass=None, - q=None): + q=None, scalar_quantities=None, vector_quantities=None): """Create :class:`ConservedVars` from separated conserved quantities.""" + if scalar_quantities is not None: + return split_conserved(dim, q=scalar_quantities) + if vector_quantities is not None: + return split_conserved(dim, q=vector_quantities) if q is not None: return split_conserved(dim, q=q) if mass is None or energy is None or momentum is None: diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 9eedb67c7..0cb646b98 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -9,6 +9,8 @@ .. autofunction:: write_visfile .. autofunction:: sim_checkpoint .. autofunction:: write_restart_file +.. autofunction:: make_fluid_restart_state +.. autofunction:: read_restart_data Diagnostic utilities -------------------- @@ -54,11 +56,25 @@ from mirgecom.io import make_status_message from mirgecom.inviscid import get_inviscid_timestep # bad smell? from meshmode.dof_array import thaw, flatten, unflatten # noqa -from mirgecom.fluid import ConservedVars # noqa +from mirgecom.fluid import make_conserved logger = logging.getLogger(__name__) +def read_restart_data(filename): + """Read the raw restart data dictionary from the given pickle restart file.""" + with open(filename, "rb") as f: + restart_data = pickle.load(f) + return restart_data + + +def make_fluid_restart_state(actx, discr, restart_q): + """Make a :class:`~mirgecom.fluid.ConservedVars` from pickled restart data.""" + from pytools.obj_array import obj_array_vectorize + q = unflatten(actx, discr, obj_array_vectorize(actx.from_numpy, restart_q)) + return make_conserved(discr.dim, q=q) + + def write_restart_file(actx, restart_dictionary, filename): """Pickle the simulation data into a file for use in restarting.""" from pytools.obj_array import obj_array_vectorize From 146cf6f2c67fe273acc85543f36f55911e390975 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 16 Jun 2021 14:42:08 -0500 Subject: [PATCH 169/385] Sharpen doc slightly --- mirgecom/inviscid.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index bbcd85726..b14d2cad3 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -67,7 +67,9 @@ def inviscid_flux(discr, eos, cv): def get_inviscid_timestep(discr, eos, cv): - """Routine returns the node-local maximum stable inviscid timestep. + """Routine returns the node-local maximum stable dt for inviscid fluid. + + The maximum stable timestep is computed from the acoustic wavespeed. Parameters ---------- From ce3373ef6b50f320b5c5c14fd4d3c785ed8b1025 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 16 Jun 2021 14:42:38 -0500 Subject: [PATCH 170/385] Sharpen doc slightly, and succintify routine a bit --- mirgecom/simutil.py | 45 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index d0446fa21..d4cd72de5 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -64,18 +64,51 @@ def check_step(step, interval): def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, t_final, constant_cfl=False): - """Return the maximum stable dt.""" + """Return the maximum stable dt for inviscid fluid simulation. + + This routine returns *dt*, the users defined constant timestep, or + *max_dt*, the maximum domain-wide stability-limited + timestep for an inviscid fluid simulation. It calls the collective: + :func:`~grudge.op.nodal_min` on the inside which makes it + domain-wide regardless of parallel decomposition. + + Two modes are supported: + - Constant DT mode: returns the minimum of (t_final-t, dt) + - Constant CFL mode: returns (cfl * max_dt) + + Parameters + ---------- + discr + Grudge discretization or discretization collection? + state: :class:`~mirgecom.fluid.ConservedVars` + The fluid state. + t: float + Current time + t_final: float + Final time + dt: float + The current timestep + cfl: float + The current CFL number + eos: :class:`~mirgecom.eos.GasEOS` + Gas equation-of-state supporting speed_of_sound + constant_cfl: bool + True if running constant CFL mode + + Returns + ------- + float + The maximum stable DT based on inviscid fluid acoustic wavespeed. + """ mydt = dt - t_remaining = t_final - t - if t_remaining < dt: - return max(0, t_remaining) - if constant_cfl is True: + t_remaining = max(0, t_final - t) + if constant_cfl: from grudge.op import nodal_min mydt = cfl * nodal_min( discr, "vol", get_inviscid_timestep(discr=discr, eos=eos, cv=state) ) - return mydt + return min(t_remaining, mydt) class ExactSolutionMismatch(Exception): From b4779d486ef14650b121804223dc0dec51e54dfa Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 16 Jun 2021 16:13:42 -0500 Subject: [PATCH 171/385] Add constant CFL example --- examples/mixture-fixed-cfl-mpi.py | 313 ++++++++++++++++++++++++++++++ mirgecom/steppers.py | 26 +-- 2 files changed, 327 insertions(+), 12 deletions(-) create mode 100644 examples/mixture-fixed-cfl-mpi.py diff --git a/examples/mixture-fixed-cfl-mpi.py b/examples/mixture-fixed-cfl-mpi.py new file mode 100644 index 000000000..816dbe496 --- /dev/null +++ b/examples/mixture-fixed-cfl-mpi.py @@ -0,0 +1,313 @@ +"""Demonstrate combustive mixture with constant CFL mode.""" + +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" +import logging +import numpy as np +import pyopencl as cl +import pyopencl.tools as cl_tools +from functools import partial + +from meshmode.array_context import PyOpenCLArrayContext +from meshmode.dof_array import thaw +from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa +from grudge.eager import EagerDGDiscretization +from grudge.shortcuts import make_visualizer + + +from mirgecom.euler import euler_operator +from mirgecom.simutil import ( + check_step, + generate_and_distribute_mesh, + ExactSolutionMismatch +) +from mirgecom.inviscid import ( + get_inviscid_timestep, + get_inviscid_cfl +) +from mirgecom.io import make_init_message +from mirgecom.mpi import mpi_entry_point +from mirgecom.integrators import rk4_step +from mirgecom.steppers import advance_state +from mirgecom.boundary import AdiabaticSlipBoundary +from mirgecom.initializers import MixtureInitializer +from mirgecom.eos import PyrometheusMixture + +import cantera +import pyrometheus as pyro + +logger = logging.getLogger(__name__) + + +@mpi_entry_point +def main(ctx_factory=cl.create_some_context, use_leap=False): + """Drive example.""" + cl_ctx = ctx_factory() + queue = cl.CommandQueue(cl_ctx) + actx = PyOpenCLArrayContext(queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + dim = 2 + nel_1d = 8 + order = 1 + + # This example runs only 3 steps by default (to keep CI ~short) + # With the mixture defined below, equilibrium is achieved at ~40ms + # To run to equlibrium, set t_final >= 40ms. + t_final = 1e-7 + current_cfl = 0.01 + velocity = np.zeros(shape=(dim,)) + current_dt = 1e-9 + current_t = 0 + constant_cfl = True + nstatus = 1 + nviz = 2 + rank = 0 + checkpoint_t = current_t + current_step = 0 + if use_leap: + from leap.rk import RK4MethodBuilder + timestepper = RK4MethodBuilder("state") + else: + timestepper = rk4_step + box_ll = -0.005 + box_ur = 0.005 + error_state = False + debug = False + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + + from meshmode.mesh.generation import generate_regular_rect_mesh + generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,) * dim, + b=(box_ur,) * dim, nelements_per_axis=(nel_1d,) * dim) + local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) + local_nelements = local_mesh.nelements + + discr = EagerDGDiscretization( + actx, local_mesh, order=order, mpi_communicator=comm + ) + nodes = thaw(actx, discr.nodes()) + + # {{{ Set up initial state using Cantera + + # Use Cantera for initialization + # -- Pick up a CTI for the thermochemistry config + # --- Note: Users may add their own CTI file by dropping it into + # --- mirgecom/mechanisms alongside the other CTI files. + from mirgecom.mechanisms import get_mechanism_cti + mech_cti = get_mechanism_cti("uiuc") + + cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti) + nspecies = cantera_soln.n_species + + # Initial temperature, pressure, and mixutre mole fractions are needed to + # set up the initial state in Cantera. + init_temperature = 1500.0 # Initial temperature hot enough to burn + # Parameters for calculating the amounts of fuel, oxidizer, and inert species + equiv_ratio = 1.0 + ox_di_ratio = 0.21 + stoich_ratio = 3.0 + # Grab the array indices for the specific species, ethylene, oxygen, and nitrogen + i_fu = cantera_soln.species_index("C2H4") + i_ox = cantera_soln.species_index("O2") + i_di = cantera_soln.species_index("N2") + x = np.zeros(nspecies) + # Set the species mole fractions according to our desired fuel/air mixture + x[i_fu] = (ox_di_ratio*equiv_ratio)/(stoich_ratio+ox_di_ratio*equiv_ratio) + x[i_ox] = stoich_ratio*x[i_fu]/equiv_ratio + x[i_di] = (1.0-ox_di_ratio)*x[i_ox]/ox_di_ratio + # Uncomment next line to make pylint fail when it can't find cantera.one_atm + one_atm = cantera.one_atm # pylint: disable=no-member + # one_atm = 101325.0 + + # Let the user know about how Cantera is being initilized + print(f"Input state (T,P,X) = ({init_temperature}, {one_atm}, {x}") + # Set Cantera internal gas temperature, pressure, and mole fractios + cantera_soln.TPX = init_temperature, one_atm, x + # Pull temperature, total density, mass fractions, and pressure from Cantera + # We need total density, and mass fractions to initialize the fluid/gas state. + can_t, can_rho, can_y = cantera_soln.TDY + can_p = cantera_soln.P + # *can_t*, *can_p* should not differ (significantly) from user's initial data, + # but we want to ensure that we use exactly the same starting point as Cantera, + # so we use Cantera's version of these data. + + # }}} + + # {{{ Create Pyrometheus thermochemistry object & EOS + + # Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and + # generates a set of methods to calculate chemothermomechanical properties and + # states for this particular mechanism. + casename = "mixture-adaptive" + pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np) + eos = PyrometheusMixture(pyrometheus_mechanism, + temperature_guess=init_temperature) + + # }}} + + # {{{ MIRGE-Com state initialization + + # Initialize the fluid/gas state with Cantera-consistent data: + # (density, pressure, temperature, mass_fractions) + print(f"Cantera state (rho,T,P,Y) = ({can_rho}, {can_t}, {can_p}, {can_y}") + initializer = MixtureInitializer(dim=dim, nspecies=nspecies, + pressure=can_p, temperature=can_t, + massfractions=can_y, velocity=velocity) + + my_boundary = AdiabaticSlipBoundary() + boundaries = {BTAG_ALL: my_boundary} + current_state = initializer(eos=eos, x_vec=nodes, t=0) + + # Inspection at physics debugging time + if debug: + print("Initial MIRGE-Com state:") + print(f"{current_state=}") + print(f"Initial DV pressure: {eos.pressure(current_state)}") + print(f"Initial DV temperature: {eos.temperature(current_state)}") + + # }}} + + visualizer = make_visualizer(discr) + initname = initializer.__class__.__name__ + eosname = eos.__class__.__name__ + init_message = make_init_message(dim=dim, order=order, + nelements=local_nelements, + global_nelements=global_nelements, + dt=current_dt, t_final=t_final, nstatus=nstatus, + nviz=nviz, cfl=current_cfl, + constant_cfl=constant_cfl, initname=initname, + eosname=eosname, casename=casename) + + # Cantera equilibrate calculates the expected end state @ chemical equilibrium + # i.e. the expected state after all reactions + cantera_soln.equilibrate("UV") + eq_temperature, eq_density, eq_mass_fractions = cantera_soln.TDY + eq_pressure = cantera_soln.P + + # Report the expected final state to the user + if rank == 0: + logger.info(init_message) + logger.info(f"Expected equilibrium state:" + f" {eq_pressure=}, {eq_temperature=}," + f" {eq_density=}, {eq_mass_fractions=}") + + def my_rhs(t, state): + return (euler_operator(discr, cv=state, t=t, + boundaries=boundaries, eos=eos) + + eos.get_species_source_terms(state)) + + def mixture_prestep_function(step, t, dt, state): + do_viz = check_step(step, nviz) + viz_fields = [("cv", state)] + current_dt = dt + + if constant_cfl: # no matter what + local_dt = get_inviscid_timestep(discr, eos=eos, cv=state) + from grudge.op import nodal_min + current_dt = current_cfl * nodal_min(discr, "vol", local_dt) + elif do_viz: # only if visualizing + local_cfl = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state) + if do_viz: # extend viz field if viz time + dv = eos.dependent_vars(state) + viz_fields.append(("dv", dv)) + # only if vizzing, calculate reaction rates + reaction_rates = eos.get_production_rates(state) + viz_fields.append(("reaction_rates", reaction_rates)) + if constant_cfl: + viz_fields.append(("dt", local_dt)) + else: + viz_fields.append(("cfl", local_cfl)) + + errors = current_dt < 0 or np.isnan(current_dt) or current_dt == np.inf + + if do_viz or errors: + from mirgecom.io import make_rank_fname, make_par_fname + rank_fn = make_rank_fname(basename=casename, rank=rank, step=step, t=t) + visualizer.write_parallel_vtk_file( + comm, rank_fn, viz_fields, overwrite=True, + par_manifest_filename=make_par_fname( + basename=casename, step=step, t=t + ) + ) + + if check_step(step, nstatus) or errors: + if not do_viz: + dv = eos.dependent_vars(state) + from grudge.op import nodal_max + min_temperature = nodal_min(discr, "vol", dv.temperature) + max_temperature = nodal_max(discr, "vol", dv.temperature) + min_pressure = nodal_min(discr, "vol", dv.pressure) + max_pressure = nodal_max(discr, "vol", dv.pressure) + if rank == 0: + logger.info(f"\nStep:{step}, Time:{t}, DT:{current_dt}," + f"CFL:{current_cfl}\n" + f"---- P({min_pressure}, {max_pressure}) " + f"T({min_temperature}, {max_temperature})\n") + + # this exit is safe, errors(current_dt) is already collective + if errors: + logger.info("Fatal error: Invalid simulation DT") + import sys + sys.exit() + + t_remaining = max(0, t_final - t) + return state, min(t_remaining, current_dt) + + try: + (current_step, current_t, current_state) = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + checkpoint=mixture_prestep_function, state=current_state, + t=current_t, t_final=t_final) + + except ExactSolutionMismatch as ex: + error_state = True + current_step = ex.step + current_t = ex.t + current_state = ex.state + + if not check_step(current_step, nviz): # If final step not an output step + if rank == 0: + logger.info("Checkpointing final state ...") + mixture_prestep_function(current_step, t=current_t, + dt=(current_t - checkpoint_t), + state=current_state) + + if current_t - t_final < 0: + error_state = True + + if error_state: + raise ValueError("Simulation did not complete successfully.") + + if rank == 0: + logger.info(f"Simulation finished at time {current_t=}.") + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main(use_leap=False) + +# vim: foldmethod=marker diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index deb54f45c..adf32dfd0 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -32,8 +32,9 @@ from mirgecom.logging_quantities import set_sim_state -def _advance_state_stepper_func(rhs, timestepper, checkpoint, get_timestep, - state, t_final, t=0.0, istep=0, logmgr=None, eos=None, dim=None): +def _advance_state_stepper_func(rhs, timestepper, t_final, state, t=0.0, istep=0, + dt=0, get_timestep=None, logmgr=None, eos=None, + checkpoint=None, dim=None): """Advance state from some time (t) to some time (t_final). Parameters @@ -80,12 +81,13 @@ def _advance_state_stepper_func(rhs, timestepper, checkpoint, get_timestep, if logmgr: logmgr.tick_before() - dt = get_timestep(state=state) - if dt < 0: - return istep, t, state + if get_timestep: + dt = get_timestep(state=state) + if dt < 0: + return istep, t, state if checkpoint: - checkpoint(state=state, step=istep, t=t, dt=dt) + state, dt = checkpoint(state=state, step=istep, t=t, dt=dt) state = timestepper(state=state, t=t, dt=dt, rhs=rhs) @@ -212,9 +214,9 @@ def generate_singlerate_leap_advancer(timestepper, component_id, rhs, t, dt, return stepper_cls -def advance_state(rhs, timestepper, checkpoint, get_timestep, state, t_final, +def advance_state(rhs, timestepper, state, t_final, dt=0, component_id="state", t=0.0, istep=0, logmgr=None, - eos=None, dim=None): + eos=None, dim=None, checkpoint=None, get_timestep=None): """Determine what stepper we're using and advance the state from (t) to (t_final). Parameters @@ -281,9 +283,9 @@ def advance_state(rhs, timestepper, checkpoint, get_timestep, state, t_final, else: (current_step, current_t, current_state) = \ _advance_state_stepper_func(rhs=rhs, timestepper=timestepper, - checkpoint=checkpoint, - get_timestep=get_timestep, state=state, - t=t, t_final=t_final, istep=istep, - logmgr=logmgr, eos=eos, dim=dim) + checkpoint=checkpoint, dt=dt, + get_timestep=get_timestep, state=state, + t=t, t_final=t_final, istep=istep, + logmgr=logmgr, eos=eos, dim=dim) return current_step, current_t, current_state From 337b647cc7a51144d6eaca03869e030913cc3db6 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 16 Jun 2021 16:16:48 -0500 Subject: [PATCH 172/385] Tweak status message just a bit... logging will take over soon. --- examples/mixture-fixed-cfl-mpi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/mixture-fixed-cfl-mpi.py b/examples/mixture-fixed-cfl-mpi.py index 816dbe496..38c0ec772 100644 --- a/examples/mixture-fixed-cfl-mpi.py +++ b/examples/mixture-fixed-cfl-mpi.py @@ -265,8 +265,8 @@ def mixture_prestep_function(step, t, dt, state): if rank == 0: logger.info(f"\nStep:{step}, Time:{t}, DT:{current_dt}," f"CFL:{current_cfl}\n" - f"---- P({min_pressure}, {max_pressure}) " - f"T({min_temperature}, {max_temperature})\n") + f"---- P({min_pressure}, {max_pressure})\n" + f"---- T({min_temperature}, {max_temperature})\n") # this exit is safe, errors(current_dt) is already collective if errors: From 0c9c459bb8f3d76e6df8a5c5544831b6f9cb15d0 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 16 Jun 2021 16:39:04 -0500 Subject: [PATCH 173/385] Make sim_checkpoint return state, dt --- mirgecom/simutil.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index d4cd72de5..58c8c2fa9 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -134,7 +134,7 @@ def sim_checkpoint(discr, visualizer, eos, cv, vizname, exact_soln=None, do_viz = check_step(step=step, interval=nviz) do_status = check_step(step=step, interval=nstatus) if do_viz is False and do_status is False: - return 0 + return cv, dt dependent_vars = eos.dependent_vars(cv) @@ -196,6 +196,8 @@ def sim_checkpoint(discr, visualizer, eos, cv, vizname, exact_soln=None, if maxerr > exittol: raise ExactSolutionMismatch(step, t=t, state=cv) + return cv, dt + def generate_and_distribute_mesh(comm, generate_mesh): """Generate a mesh and distribute it among all ranks in *comm*. From b4c1f27e5248c85325d0889ac3e5c08818b30b76 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 17 Jun 2021 11:01:23 -0500 Subject: [PATCH 174/385] Add user-defined quantity logging from #391 for production use. --- mirgecom/logging_quantities.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/mirgecom/logging_quantities.py b/mirgecom/logging_quantities.py index 08320e764..3b280158c 100644 --- a/mirgecom/logging_quantities.py +++ b/mirgecom/logging_quantities.py @@ -227,6 +227,30 @@ def set_state_vars(self, state_vars: np.ndarray) -> None: # }}} +# {{{ User-defined quantities + + +class LogUserQuantity(LogQuantity): + """Logging support for arbitrary user quantities.""" + + def __init__(self, name="user_quantity", value=None, user_function=None) -> None: + """Initialize the user's log quantity.""" + LogQuantity.__init__(self, name, "1") + self._value = value + self._uf = user_function + + def set_quantity(self, value) -> None: + """Set the user quantity to be used in calculating the logged value.""" + self._value = value + + def __call__(self) -> float: + """Return the value of cfl.""" + if self._uf: + return self._uf(self._value) + return self._value + +# }}} + # {{{ Discretization-based quantities From 43dd81866c7d40c1df76e4c4e038118af269b871 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 21 Jun 2021 10:49:57 -0500 Subject: [PATCH 175/385] Add restart module for production --- mirgecom/restart.py | 58 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 mirgecom/restart.py diff --git a/mirgecom/restart.py b/mirgecom/restart.py new file mode 100644 index 000000000..d83b58fa5 --- /dev/null +++ b/mirgecom/restart.py @@ -0,0 +1,58 @@ +"""Provide some utilities for restarting simulations. + +.. autofunction:: read_restart_data +.. autofunction:: write_restart_file +.. autofunction:: make_fluid_state + +""" + +__copyright__ = """ +Copyright (C) 2021 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import pickle +from meshmode.dof_array import array_context_for_pickling + + +def read_restart_data(actx, filename): + """Read the raw restart data dictionary from the given pickle restart file.""" + with array_context_for_pickling(actx): + with open(filename, "rb") as f: + return pickle.load(f) + + +def write_restart_file(actx, restart_data, filename, comm=None): + """Pickle the simulation data into a file for use in restarting.""" + rank = 0 + if comm: + rank = comm.Get_rank() + if rank == 0: + import os + rst_dir = os.path.dirname(filename) + if not os.path.exists(rst_dir): + os.makedirs(rst_dir) + if comm: + comm.barrier() + with array_context_for_pickling(actx): + with open(filename, "wb") as f: + pickle.dump(restart_data, f) From 731ee2ca9c65331d041674124a514f3b08f6e2f9 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 21 Jun 2021 11:03:45 -0500 Subject: [PATCH 176/385] Add restart capability to autoignition --- examples/autoignition-mpi.py | 62 +++++++++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index bcbe668c4..41099e8c9 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -60,7 +60,8 @@ @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False): +def main(ctx_factory=cl.create_some_context, casename="autoignition", use_leap=False, + restart_step=None, restart_name=None): """Drive example.""" cl_ctx = ctx_factory() queue = cl.CommandQueue(cl_ctx) @@ -82,6 +83,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 1 nviz = 5 + nrestart = 5 rank = 0 checkpoint_t = current_t current_step = 0 @@ -98,12 +100,31 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() - - from meshmode.mesh.generation import generate_regular_rect_mesh - generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,) * dim, - b=(box_ur,) * dim, nelements_per_axis=(nel_1d,) * dim) - local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) - local_nelements = local_mesh.nelements + nproc = comm.Get_size() + + restart_file_pattern = "{casename}-{step:04d}-{rank:04d}.pkl" + restart_path = "restart_data/" + if restart_step: + if not restart_name: + restart_name = casename + rst_filename = ( + restart_path + + restart_file_pattern.format(casename=restart_name, + step=restart_step, rank=rank) + ) + from mirgecom.restart import read_restart_data + restart_data = read_restart_data(actx, rst_filename) + local_mesh = restart_data["local_mesh"] + local_nelements = local_mesh.nelements + global_nelements = restart_data["global_nelements"] + assert restart_data["nparts"] == nproc + else: + from meshmode.mesh.generation import generate_regular_rect_mesh + generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, + b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) + local_mesh, global_nelements = generate_and_distribute_mesh(comm, + generate_mesh) + local_nelements = local_mesh.nelements discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm @@ -179,7 +200,13 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): my_boundary = AdiabaticSlipBoundary() boundaries = {BTAG_ALL: my_boundary} - current_state = initializer(eos=eos, x_vec=nodes, time=0) + if restart_step: + current_t = restart_data["t"] + current_step = restart_step + current_state = restart_data["state"] + else: + # Set the current state from time 0 + current_state = initializer(eos=eos, x_vec=nodes, time=0) # Inspection at physics debugging time if debug: @@ -224,6 +251,25 @@ def my_rhs(t, state): + eos.get_species_source_terms(state)) def my_checkpoint(step, t, dt, state): + if check_step(step, nrestart) and step != restart_step: + rst_filename = ( + restart_path + + restart_file_pattern.format(casename=casename, step=step, + rank=rank) + ) + rst_data = { + "local_mesh": local_mesh, + "state": current_state, + "t": t, + "step": step, + "global_nelements": global_nelements, + "num_parts": nproc + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_filename, comm) + + # awful - computes potentially expensive viz quantities + # regardless of whether it is time to viz reaction_rates = eos.get_production_rates(state) local_cfl = get_inviscid_cfl(discr, eos=eos, dt=current_dt, cv=state) viz_fields = [("reaction_rates", reaction_rates), From df4995da29ccd51aabf062cabd8bff170f2ff6c6 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 21 Jun 2021 11:05:12 -0500 Subject: [PATCH 177/385] Rip out old restart interface. --- mirgecom/simutil.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 0cb646b98..28168cd65 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -8,9 +8,6 @@ .. autoexception:: ExactSolutionMismatch .. autofunction:: write_visfile .. autofunction:: sim_checkpoint -.. autofunction:: write_restart_file -.. autofunction:: make_fluid_restart_state -.. autofunction:: read_restart_data Diagnostic utilities -------------------- @@ -49,42 +46,15 @@ THE SOFTWARE. """ import logging -import pickle import numpy as np from meshmode.dof_array import thaw from mirgecom.io import make_status_message from mirgecom.inviscid import get_inviscid_timestep # bad smell? -from meshmode.dof_array import thaw, flatten, unflatten # noqa -from mirgecom.fluid import make_conserved logger = logging.getLogger(__name__) -def read_restart_data(filename): - """Read the raw restart data dictionary from the given pickle restart file.""" - with open(filename, "rb") as f: - restart_data = pickle.load(f) - return restart_data - - -def make_fluid_restart_state(actx, discr, restart_q): - """Make a :class:`~mirgecom.fluid.ConservedVars` from pickled restart data.""" - from pytools.obj_array import obj_array_vectorize - q = unflatten(actx, discr, obj_array_vectorize(actx.from_numpy, restart_q)) - return make_conserved(discr.dim, q=q) - - -def write_restart_file(actx, restart_dictionary, filename): - """Pickle the simulation data into a file for use in restarting.""" - from pytools.obj_array import obj_array_vectorize - state = restart_dictionary["state"].join() - restart_dictionary["state"] = obj_array_vectorize(actx.to_numpy, - flatten(state)) - with open(filename, "wb") as f: - pickle.dump(restart_dictionary, f) - - def check_step(step, interval): """ Check step number against a user-specified interval. From 93379b8ad148de2f5e6859a25ffd9616a9506d7d Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 21 Jun 2021 12:10:15 -0500 Subject: [PATCH 178/385] Do not create directory unless pathname is non-empty. --- mirgecom/restart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index d83b58fa5..455ea1fde 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -49,7 +49,7 @@ def write_restart_file(actx, restart_data, filename, comm=None): if rank == 0: import os rst_dir = os.path.dirname(filename) - if not os.path.exists(rst_dir): + if rst_dir and not os.path.exists(rst_dir): os.makedirs(rst_dir) if comm: comm.barrier() From d3256a0322677e3fc3a3ac5f313d3ec604963dbe Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 21 Jun 2021 12:57:30 -0500 Subject: [PATCH 179/385] Create viz directory if it dne. --- mirgecom/simutil.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 28168cd65..47e3b520f 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -122,11 +122,19 @@ def write_visfile(discr, io_fields, visualizer, vizname, comm = discr.mpi_communicator rank = 0 - if comm is not None: + if comm: rank = comm.Get_rank() rank_fn = make_rank_fname(basename=vizname, rank=rank, step=step, t=t) + if rank == 0: + import os + viz_dir = os.path.dirname(rank_fn) + if viz_dir and not os.path.exists(viz_dir): + os.makedirs(viz_dir) + if comm: + comm.barrier() + if vis_timer: ctm = vis_timer.start_sub_timer() else: From 49e0c56d7fac926f3e812a8c9b5e1e7a89001fa8 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Mon, 21 Jun 2021 18:44:49 -0700 Subject: [PATCH 180/385] Memoize DV on CV --- mirgecom/eos.py | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/mirgecom/eos.py b/mirgecom/eos.py index 920813649..7d282d9ec 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -38,6 +38,7 @@ from dataclasses import dataclass import numpy as np +from pytools import memoize_in from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from mirgecom.fluid import ConservedVars, make_conserved @@ -428,9 +429,19 @@ def pressure(self, cv: ConservedVars): p = (\gamma_{\mathtt{mix}} - 1)e """ - temperature = self.temperature(cv) - y = self.species_fractions(cv) - return self._pyrometheus_mech.get_pressure(cv.mass, temperature, y) + @memoize_in(cv, (PyrometheusMixture.pressure, + type(self._pyrometheus_mech))) + def get(): + temperature = self.temperature(cv) + y = self.species_fractions(cv) + press = self._pyrometheus_mech.get_pressure(cv.mass, temperature, y) + # from meshmode.dof_array import freeze + # return freeze(cv.array_context, press) + return press + + # from meshmode.dof_array import thaw + # return thaw(cv.array_context, get()) + return get() def sound_speed(self, cv: ConservedVars): r"""Get the speed of sound in the gas. @@ -445,6 +456,7 @@ def sound_speed(self, cv: ConservedVars): c2 = (self.gamma(cv) * self.pressure(cv)) / cv.mass return actx.np.sqrt(c2) + def temperature(self, cv: ConservedVars): r"""Get the thermodynamic temperature of the gas. @@ -456,9 +468,20 @@ def temperature(self, cv: ConservedVars): T = \frac{(\gamma_{\mathtt{mix}} - 1)e}{R_s \rho} """ - y = self.species_fractions(cv) - e = self.internal_energy(cv) / cv.mass - return self._pyrometheus_mech.get_temperature(e, self._tguess, y, True) + @memoize_in(cv, (PyrometheusMixture.temperature, + type(self._pyrometheus_mech))) + def get(): + y = self.species_fractions(cv) + e = self.internal_energy(cv) / cv.mass + tmptr = self._pyrometheus_mech.get_temperature(e, self._tguess, + y, True) + # from meshmode.dof_array import freeze + # return freeze(cv.array_context, tmptr) + return tmptr + + # from meshmode.dof_array import thaw + # return thaw(cv.array_context, get()) + return get() def total_energy(self, cv, pressure): r""" From 5f5798d4aa712a06d1e5ebd63e50af8824ac5317 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Mon, 21 Jun 2021 19:17:49 -0700 Subject: [PATCH 181/385] Override __reduce__ so CV will pickle correctly without cached items. --- mirgecom/fluid.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mirgecom/fluid.py b/mirgecom/fluid.py index 5fadbcd3f..19653631c 100644 --- a/mirgecom/fluid.py +++ b/mirgecom/fluid.py @@ -42,7 +42,7 @@ import numpy as np # noqa from pytools.obj_array import make_obj_array from meshmode.dof_array import DOFArray # noqa -from dataclasses import dataclass +from dataclasses import dataclass, fields from arraycontext import ( dataclass_array_container, with_container_arithmetic, @@ -239,6 +239,11 @@ def dim(self): """Return the number of physical dimensions.""" return len(self.momentum) + def __reduce__(self): + """Return a tuple reproduction of self for pickling.""" + return(ConservedVars, tuple(getattr(self, f.name) + for f in fields(ConservedVars))) + def join(self): """Call :func:`join_conserved` on *self*.""" return join_conserved( @@ -254,6 +259,7 @@ def replace(self, **kwargs): return replace(self, **kwargs) + def _aux_shape(ary, leading_shape): """:arg leading_shape: a tuple with which ``ary.shape`` is expected to begin.""" from meshmode.dof_array import DOFArray From 5ab68d9743b3796e79b3d8e0c4d0b4e82151b2ad Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Mon, 21 Jun 2021 19:36:29 -0700 Subject: [PATCH 182/385] Placate flake8 --- mirgecom/eos.py | 1 - mirgecom/fluid.py | 1 - 2 files changed, 2 deletions(-) diff --git a/mirgecom/eos.py b/mirgecom/eos.py index 7d282d9ec..9a16d6c08 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -456,7 +456,6 @@ def sound_speed(self, cv: ConservedVars): c2 = (self.gamma(cv) * self.pressure(cv)) / cv.mass return actx.np.sqrt(c2) - def temperature(self, cv: ConservedVars): r"""Get the thermodynamic temperature of the gas. diff --git a/mirgecom/fluid.py b/mirgecom/fluid.py index 19653631c..c03c58d88 100644 --- a/mirgecom/fluid.py +++ b/mirgecom/fluid.py @@ -259,7 +259,6 @@ def replace(self, **kwargs): return replace(self, **kwargs) - def _aux_shape(ary, leading_shape): """:arg leading_shape: a tuple with which ``ary.shape`` is expected to begin.""" from meshmode.dof_array import DOFArray From 053113e1a557c00cd64cb6b987609080232de02c Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Mon, 21 Jun 2021 19:46:38 -0700 Subject: [PATCH 183/385] Memoize DV on CV --- mirgecom/eos.py | 34 ++++++++++++++++++++++++++++------ mirgecom/fluid.py | 5 +++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/mirgecom/eos.py b/mirgecom/eos.py index 3b8008442..3383d2a16 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -38,6 +38,7 @@ from dataclasses import dataclass import numpy as np +from pytools import memoize_in from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from mirgecom.fluid import ConservedVars, make_conserved @@ -398,9 +399,19 @@ def pressure(self, cv: ConservedVars): p = (\gamma_{\mathtt{mix}} - 1)e """ - temperature = self.temperature(cv) - y = self.species_fractions(cv) - return self._pyrometheus_mech.get_pressure(cv.mass, temperature, y) + @memoize_in(cv, (PyrometheusMixture.pressure, + type(self._pyrometheus_mech))) + def get(): + temperature = self.temperature(cv) + y = self.species_fractions(cv) + press = self._pyrometheus_mech.get_pressure(cv.mass, temperature, y) + # from meshmode.dof_array import freeze + # return freeze(cv.array_context, press) + return press + + # from meshmode.dof_array import thaw + # return thaw(cv.array_context, get()) + return get() def sound_speed(self, cv: ConservedVars): r"""Get the speed of sound in the gas. @@ -426,9 +437,20 @@ def temperature(self, cv: ConservedVars): T = \frac{(\gamma_{\mathtt{mix}} - 1)e}{R_s \rho} """ - y = self.species_fractions(cv) - e = self.internal_energy(cv) / cv.mass - return self._pyrometheus_mech.get_temperature(e, self._tguess, y, True) + @memoize_in(cv, (PyrometheusMixture.temperature, + type(self._pyrometheus_mech))) + def get(): + y = self.species_fractions(cv) + e = self.internal_energy(cv) / cv.mass + tmptr = self._pyrometheus_mech.get_temperature(e, self._tguess, + y, True) + # from meshmode.dof_array import freeze + # return freeze(cv.array_context, tmptr) + return tmptr + + # from meshmode.dof_array import thaw + # return thaw(cv.array_context, get()) + return get() def total_energy(self, cv, pressure): r""" diff --git a/mirgecom/fluid.py b/mirgecom/fluid.py index 21b2f5d21..97575f681 100644 --- a/mirgecom/fluid.py +++ b/mirgecom/fluid.py @@ -248,6 +248,11 @@ def join(self): momentum=self.momentum, species_mass=self.species_mass) + def __reduce__(self): + """Return a tuple reproduction of self for pickling.""" + return(ConservedVars, tuple(getattr(self, f.name) + for f in fields(ConservedVars))) + def replace(self, **kwargs): """Return a copy of *self* with the attributes in *kwargs* replaced.""" from dataclasses import replace From c3295afaca14549ba49ced264bf99d31453131e5 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Mon, 21 Jun 2021 19:51:41 -0700 Subject: [PATCH 184/385] Add missing dataclasses.fields --- mirgecom/fluid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/fluid.py b/mirgecom/fluid.py index 97575f681..c859217a6 100644 --- a/mirgecom/fluid.py +++ b/mirgecom/fluid.py @@ -42,7 +42,7 @@ import numpy as np # noqa from pytools.obj_array import make_obj_array from meshmode.dof_array import DOFArray # noqa -from dataclasses import dataclass +from dataclasses import dataclass, fields from arraycontext import ( dataclass_array_container, with_container_arithmetic, From 0a01f74a5ac5b60ba0874710c54ffa705701df54 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 22 Jun 2021 09:20:15 -0500 Subject: [PATCH 185/385] Memoize more stuff --- mirgecom/eos.py | 35 ++++++++++++++++++++++++----------- mirgecom/fluid.py | 19 +++++++++++++++---- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/mirgecom/eos.py b/mirgecom/eos.py index 3383d2a16..3b84d3f89 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -178,7 +178,10 @@ def pressure(self, cv: ConservedVars): p = (\gamma - 1)e """ - return self.internal_energy(cv) * (self._gamma - 1.0) + @memoize_in(cv, (IdealSingleGas.pressure, "pressarry")) + def get(): + return self.internal_energy(cv) * (self._gamma - 1.0) + return get() def sound_speed(self, cv: ConservedVars): r"""Get the speed of sound in the gas. @@ -191,9 +194,12 @@ def sound_speed(self, cv: ConservedVars): """ actx = cv.mass.array_context - p = self.pressure(cv) - c2 = self._gamma / cv.mass * p - return actx.np.sqrt(c2) + @memoize_in(cv, (IdealSingleGas.sound_speed, "sosarry")) + def get(): + p = self.pressure(cv) + c2 = self._gamma / cv.mass * p + return actx.np.sqrt(c2) + return get() def temperature(self, cv: ConservedVars): r"""Get the thermodynamic temperature of the gas. @@ -206,10 +212,13 @@ def temperature(self, cv: ConservedVars): T = \frac{(\gamma - 1)e}{R\rho} """ - return ( - (((self._gamma - 1.0) / self._gas_const) - * self.internal_energy(cv) / cv.mass) - ) + @memoize_in(cv, (IdealSingleGas.temperature, "temparry")) + def get(): + return ( + (((self._gamma - 1.0) / self._gas_const) + * self.internal_energy(cv) / cv.mass) + ) + return get() def total_energy(self, cv, pressure): r""" @@ -422,9 +431,13 @@ def sound_speed(self, cv: ConservedVars): c = \sqrt{\frac{\gamma_{\mathtt{mix}}{p}}{\rho}} """ - actx = cv.mass.array_context - c2 = (self.gamma(cv) * self.pressure(cv)) / cv.mass - return actx.np.sqrt(c2) + @memoize_in(cv, (PyrometheusMixture.sound_speed, + type(self._pyrometheus_mech))) + def get(): + actx = cv.mass.array_context + c2 = (self.gamma(cv) * self.pressure(cv)) / cv.mass + return actx.np.sqrt(c2) + return get() def temperature(self, cv: ConservedVars): r"""Get the thermodynamic temperature of the gas. diff --git a/mirgecom/fluid.py b/mirgecom/fluid.py index c859217a6..fa3f60360 100644 --- a/mirgecom/fluid.py +++ b/mirgecom/fluid.py @@ -41,6 +41,7 @@ """ import numpy as np # noqa from pytools.obj_array import make_obj_array +from pytools import memoize_in from meshmode.dof_array import DOFArray # noqa from dataclasses import dataclass, fields from arraycontext import ( @@ -239,6 +240,14 @@ def dim(self): """Return the number of physical dimensions.""" return len(self.momentum) + @property + def velocity(self): + """Return the fluid velocity = momentum / mass.""" + @memoize_in(self, "vel_arry") + def get(): + return self.momentum / self.mass + return get() + def join(self): """Call :func:`join_conserved` on *self*.""" return join_conserved( @@ -429,7 +438,9 @@ def compute_wavespeed(dim, eos, cv: ConservedVars): where $\mathbf{v}$ is the flow velocity and c is the speed of sound in the fluid. """ - actx = cv.array_context - - v = cv.momentum / cv.mass - return actx.np.sqrt(np.dot(v, v)) + eos.sound_speed(cv) + @memoize_in(cv, ("wvspd_arry", type(eos))) + def get(): + actx = cv.array_context + v = cv.velocity + return actx.np.sqrt(np.dot(v, v)) + eos.sound_speed(cv) + return get() From 4c94a7d90298e574112592c8efadaffbb471ca80 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 23 Jun 2021 09:18:50 -0500 Subject: [PATCH 186/385] Fix merge errors --- examples/autoignition-mpi.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index c132adbd5..5b97cf7ca 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -84,7 +84,9 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() + nproc = comm.Get_size() + restart_file_pattern = "{casename}-{step:04d}-{rank:04d}.pkl" logmgr = initialize_logmgr(use_logmgr, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) @@ -129,11 +131,6 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, box_ur = 0.005 debug = False - from mpi4py import MPI - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - nproc = comm.Get_size() - restart_file_pattern = "{casename}-{step:04d}-{rank:04d}.pkl" restart_path = "restart_data/" if restart_step: @@ -316,8 +313,8 @@ def my_checkpoint(step, t, dt, state, force=False): do_viz = force or check_step(step=step, interval=nviz) do_health = force or check_step(step=step, interval=nhealth) do_logstart = force or check_step(step=step, interval=nlog) - do_restart = (force or check_step(step, nrestart) and - step != restart) + do_restart = (force or check_step(step, nrestart) + and step != restart_step) if do_logstart and logmgr: logmgr.tick_before() @@ -336,7 +333,8 @@ def my_checkpoint(step, t, dt, state, force=False): "num_parts": nproc } from mirgecom.restart import write_restart_file - write_restart_file(actx, rst_data, rst_filename, comm) + write_restart_file(actx, rst_data, rst_filename, + comm=discr.mpi_communicator) if do_viz or do_health: dv = eos.dependent_vars(state) From 7c0145afe679a74d6381b1517d835b9d5d4d7fe8 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 24 Jun 2021 10:44:32 -0500 Subject: [PATCH 187/385] Remove extraneous comments, make return not look like a function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andreas Klöckner --- mirgecom/eos.py | 4 ---- mirgecom/fluid.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/mirgecom/eos.py b/mirgecom/eos.py index 3b84d3f89..3d6a511ce 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -414,12 +414,8 @@ def get(): temperature = self.temperature(cv) y = self.species_fractions(cv) press = self._pyrometheus_mech.get_pressure(cv.mass, temperature, y) - # from meshmode.dof_array import freeze - # return freeze(cv.array_context, press) return press - # from meshmode.dof_array import thaw - # return thaw(cv.array_context, get()) return get() def sound_speed(self, cv: ConservedVars): diff --git a/mirgecom/fluid.py b/mirgecom/fluid.py index fa3f60360..b843f2013 100644 --- a/mirgecom/fluid.py +++ b/mirgecom/fluid.py @@ -259,7 +259,7 @@ def join(self): def __reduce__(self): """Return a tuple reproduction of self for pickling.""" - return(ConservedVars, tuple(getattr(self, f.name) + return (ConservedVars, tuple(getattr(self, f.name) for f in fields(ConservedVars))) def replace(self, **kwargs): From 5f38088d305fbf7f7cfe41a39e31214e8aed14d6 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 24 Jun 2021 10:53:50 -0500 Subject: [PATCH 188/385] Sharpen logic/comment for clarity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andreas Klöckner --- examples/mixture-fixed-cfl-mpi.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/mixture-fixed-cfl-mpi.py b/examples/mixture-fixed-cfl-mpi.py index 38c0ec772..e861bb6ce 100644 --- a/examples/mixture-fixed-cfl-mpi.py +++ b/examples/mixture-fixed-cfl-mpi.py @@ -229,8 +229,10 @@ def mixture_prestep_function(step, t, dt, state): local_dt = get_inviscid_timestep(discr, eos=eos, cv=state) from grudge.op import nodal_min current_dt = current_cfl * nodal_min(discr, "vol", local_dt) - elif do_viz: # only if visualizing - local_cfl = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state) + else: + # constant dt + if do_viz: # only if visualizing + local_cfl = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state) if do_viz: # extend viz field if viz time dv = eos.dependent_vars(state) viz_fields.append(("dv", dv)) From 90dfda2d2b65b48af2123fa1140ee2ab4e8837ee Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 25 Jun 2021 08:38:22 -0500 Subject: [PATCH 189/385] Fix up Leap for constant cfl mode, clean up loose ends per review comments. --- examples/mixture-fixed-cfl-mpi.py | 16 ++++++++-------- mirgecom/simutil.py | 4 ++-- mirgecom/steppers.py | 30 +++++++++++++++++++----------- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/examples/mixture-fixed-cfl-mpi.py b/examples/mixture-fixed-cfl-mpi.py index e861bb6ce..a7200d2aa 100644 --- a/examples/mixture-fixed-cfl-mpi.py +++ b/examples/mixture-fixed-cfl-mpi.py @@ -225,15 +225,15 @@ def mixture_prestep_function(step, t, dt, state): viz_fields = [("cv", state)] current_dt = dt - if constant_cfl: # no matter what + if constant_cfl: local_dt = get_inviscid_timestep(discr, eos=eos, cv=state) from grudge.op import nodal_min current_dt = current_cfl * nodal_min(discr, "vol", local_dt) - else: - # constant dt - if do_viz: # only if visualizing + else: # constant dt mode + if do_viz: # calculate cfl field only if visualizing local_cfl = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state) if do_viz: # extend viz field if viz time + # Calculate DV only if needed for visualization dv = eos.dependent_vars(state) viz_fields.append(("dv", dv)) # only if vizzing, calculate reaction rates @@ -241,12 +241,12 @@ def mixture_prestep_function(step, t, dt, state): viz_fields.append(("reaction_rates", reaction_rates)) if constant_cfl: viz_fields.append(("dt", local_dt)) - else: + else: # constant dt mode viz_fields.append(("cfl", local_cfl)) errors = current_dt < 0 or np.isnan(current_dt) or current_dt == np.inf - if do_viz or errors: + if do_viz or errors: # write viz at viztime, or if there were errors from mirgecom.io import make_rank_fname, make_par_fname rank_fn = make_rank_fname(basename=casename, rank=rank, step=step, t=t) visualizer.write_parallel_vtk_file( @@ -257,7 +257,7 @@ def mixture_prestep_function(step, t, dt, state): ) if check_step(step, nstatus) or errors: - if not do_viz: + if not do_viz: # we already have dv on viz steps dv = eos.dependent_vars(state) from grudge.op import nodal_max min_temperature = nodal_min(discr, "vol", dv.temperature) @@ -277,7 +277,7 @@ def mixture_prestep_function(step, t, dt, state): sys.exit() t_remaining = max(0, t_final - t) - return state, min(t_remaining, current_dt) + return min(t_remaining, current_dt) try: (current_step, current_t, current_state) = \ diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 58c8c2fa9..55e5fb4c6 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -134,7 +134,7 @@ def sim_checkpoint(discr, visualizer, eos, cv, vizname, exact_soln=None, do_viz = check_step(step=step, interval=nviz) do_status = check_step(step=step, interval=nstatus) if do_viz is False and do_status is False: - return cv, dt + return dt dependent_vars = eos.dependent_vars(cv) @@ -196,7 +196,7 @@ def sim_checkpoint(discr, visualizer, eos, cv, vizname, exact_soln=None, if maxerr > exittol: raise ExactSolutionMismatch(step, t=t, state=cv) - return cv, dt + return dt def generate_and_distribute_mesh(comm, generate_mesh): diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index adf32dfd0..2d0bb58ed 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -83,11 +83,12 @@ def _advance_state_stepper_func(rhs, timestepper, t_final, state, t=0.0, istep=0 if get_timestep: dt = get_timestep(state=state) - if dt < 0: - return istep, t, state if checkpoint: - state, dt = checkpoint(state=state, step=istep, t=t, dt=dt) + dt = checkpoint(state=state, step=istep, t=t, dt=dt) + + if dt < 0: + return istep, t, state state = timestepper(state=state, t=t, dt=dt, rhs=rhs) @@ -102,9 +103,10 @@ def _advance_state_stepper_func(rhs, timestepper, t_final, state, t=0.0, istep=0 return istep, t, state -def _advance_state_leap(rhs, timestepper, checkpoint, get_timestep, - state, t_final, component_id="state", t=0.0, istep=0, - logmgr=None, eos=None, dim=None): +def _advance_state_leap(rhs, timestepper, checkpoint, state, t_final, dt=0, + component_id="state", t=0.0, istep=0, + logmgr=None, eos=None, dim=None, checkpoint=None, + get_timestep=None): """Advance state from some time *t* to some time *t_final* using :mod:`leap`. Parameters @@ -147,7 +149,8 @@ def _advance_state_leap(rhs, timestepper, checkpoint, get_timestep, return istep, t, state # Generate code for Leap method. - dt = get_timestep(state=state) + if get_timestep: + dt = get_timestep(state=state) stepper_cls = generate_singlerate_leap_advancer(timestepper, component_id, rhs, t, dt, state) while t < t_final: @@ -155,11 +158,16 @@ def _advance_state_leap(rhs, timestepper, checkpoint, get_timestep, if logmgr: logmgr.tick_before() - dt = get_timestep(state=state) + if get_timestep: + dt = get_timestep(state=state) + + if checkpoint: + dt = checkpoint(state=state, step=istep, t=t, dt=dt) + if dt < 0: return istep, t, state - checkpoint(state=state, step=istep, t=t, dt=dt) + stepper_cls.dt = dt # Leap interface here is *a bit* different. for event in stepper_cls.run(t_end=t+dt): @@ -276,8 +284,8 @@ def advance_state(rhs, timestepper, state, t_final, dt=0, if leap_timestepper: (current_step, current_t, current_state) = \ _advance_state_leap(rhs=rhs, timestepper=timestepper, - checkpoint=checkpoint, - get_timestep=get_timestep, state=state, + checkpoint=checkpoint,dt=0, + get_timestep=get_timestep, state=state, t=t, t_final=t_final, component_id=component_id, istep=istep, logmgr=logmgr, eos=eos, dim=dim) else: From c3169c24bea84120ba7dcc557a96990e0b611169 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 25 Jun 2021 08:48:56 -0500 Subject: [PATCH 190/385] Attempt to address linting errors. --- mirgecom/steppers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 2d0bb58ed..29411be42 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -103,7 +103,7 @@ def _advance_state_stepper_func(rhs, timestepper, t_final, state, t=0.0, istep=0 return istep, t, state -def _advance_state_leap(rhs, timestepper, checkpoint, state, t_final, dt=0, +def _advance_state_leap(rhs, timestepper, state, t_final, dt=0, component_id="state", t=0.0, istep=0, logmgr=None, eos=None, dim=None, checkpoint=None, get_timestep=None): @@ -284,7 +284,7 @@ def advance_state(rhs, timestepper, state, t_final, dt=0, if leap_timestepper: (current_step, current_t, current_state) = \ _advance_state_leap(rhs=rhs, timestepper=timestepper, - checkpoint=checkpoint,dt=0, + checkpoint=checkpoint, dt=0, get_timestep=get_timestep, state=state, t=t, t_final=t_final, component_id=component_id, istep=istep, logmgr=logmgr, eos=eos, dim=dim) From e7deb73c2636258c815dc5eff544668099ce95d9 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 25 Jun 2021 12:21:58 -0500 Subject: [PATCH 191/385] Merge with main AV development --- mirgecom/artificial_viscosity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 3fa687766..b80e34783 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -317,8 +317,8 @@ def highest_mode(grp): indicator = actx.np.log10(indicator + 1.0e-12) # Compute artificial viscosity percentage based on indicator and set parameters - yesnol = indicator > (s0 - kappa) - yesnou = indicator > (s0 + kappa) + yesnol = actx.np.greater(indicator, (s0 - kappa)) + yesnou = actx.np.greater(indicator, (s0 + kappa)) sin_indicator = actx.np.where( yesnol, 0.5 * (1.0 + actx.np.sin(np.pi * (indicator - s0) / (2.0 * kappa))), From 408411902b34409121c8cfe302155bec149599f1 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 25 Jun 2021 12:26:14 -0500 Subject: [PATCH 192/385] Merge with current AV --- mirgecom/artificial_viscosity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 3fa687766..b80e34783 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -317,8 +317,8 @@ def highest_mode(grp): indicator = actx.np.log10(indicator + 1.0e-12) # Compute artificial viscosity percentage based on indicator and set parameters - yesnol = indicator > (s0 - kappa) - yesnou = indicator > (s0 + kappa) + yesnol = actx.np.greater(indicator, (s0 - kappa)) + yesnou = actx.np.greater(indicator, (s0 + kappa)) sin_indicator = actx.np.where( yesnol, 0.5 * (1.0 + actx.np.sin(np.pi * (indicator - s0) / (2.0 * kappa))), From 708593a49096853f42d029ec1502acb6ab5aee3c Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 25 Jun 2021 18:00:44 -0500 Subject: [PATCH 193/385] Make dummy checkpoint return dt --- test/test_time_integrators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_time_integrators.py b/test/test_time_integrators.py index fe69d7514..2a4a96170 100644 --- a/test/test_time_integrators.py +++ b/test/test_time_integrators.py @@ -128,7 +128,7 @@ def get_timestep(state): return dt def my_checkpoint(state, step, t, dt): - return 0 + return dt (step, t, state) = \ advance_state(rhs=rhs, timestepper=method, From c867a12db9d2dcbf6bb15743cf496b9cfd5304ae Mon Sep 17 00:00:00 2001 From: jlevine18 Date: Sat, 26 Jun 2021 10:51:13 -0500 Subject: [PATCH 194/385] Add Viscous Timestepping (#400) Co-authored-by: Mike Campbell --- examples/nsmix-mpi.py | 53 +++++++++++++------------------- mirgecom/simutil.py | 4 ++- mirgecom/steppers.py | 25 ++++++++-------- mirgecom/viscous.py | 70 ++++++++++++++++++++++++++++++++----------- test/test_viscous.py | 57 +++++++++++++++++++++++++++++++++++ 5 files changed, 146 insertions(+), 63 deletions(-) diff --git a/examples/nsmix-mpi.py b/examples/nsmix-mpi.py index 08f51fa88..12c5bf86d 100644 --- a/examples/nsmix-mpi.py +++ b/examples/nsmix-mpi.py @@ -35,13 +35,9 @@ from grudge.eager import EagerDGDiscretization from grudge.shortcuts import make_visualizer -from mirgecom.inviscid import ( - get_inviscid_timestep -) from mirgecom.transport import SimpleTransport from mirgecom.viscous import get_viscous_timestep from mirgecom.navierstokes import ns_operator -# from mirgecom.heat import heat_operator from mirgecom.simutil import ( sim_checkpoint, @@ -81,12 +77,12 @@ def main(ctx_factory=cl.create_some_context): # This example runs only 3 steps by default (to keep CI ~short) # With the mixture defined below, equilibrium is achieved at ~40ms # To run to equlibrium, set t_final >= 40ms. - t_final = 3e-9 - current_cfl = 1.0 + t_final = 5e-9 + current_cfl = .0009 velocity = np.zeros(shape=(dim,)) current_dt = 1e-9 current_t = 0 - constant_cfl = False + constant_cfl = True nstatus = 1 nviz = 5 rank = 0 @@ -232,28 +228,6 @@ def main(ctx_factory=cl.create_some_context): f" {eq_pressure=}, {eq_temperature=}," f" {eq_density=}, {eq_mass_fractions=}") - def get_timestep(state): - next_dt = current_dt - t_end = t_final - if constant_cfl is True: - inviscid_dt = ( - current_cfl * get_inviscid_timestep(discr=discr, eos=eos, cv=state) - ) - viscous_dt = get_viscous_timestep(discr=discr, eos=eos, - transport_model=transport_model, - cfl=current_cfl, cv=state) - next_dt = min([next_dt, inviscid_dt, viscous_dt]) - # else: - # inviscid_cfl = get_inviscid_cfl(discr=discr, eos=eos, - # dt=next_dt, state=state) - # viscous_cfl = get_viscous_cfl(discr, eos=eos, - # transport_model=transport_model, - # dt=next_dt, state=state) - if(current_t + next_dt) > t_end: - next_dt = t_end - current_t - - return next_dt - def my_rhs(t, state): ns_rhs = ns_operator(discr, cv=state, t=t, boundaries=visc_bnds, eos=eos) @@ -263,8 +237,23 @@ def my_rhs(t, state): def my_checkpoint(step, t, dt, state): reaction_rates = eos.get_production_rates(state) viz_fields = [("reaction_rates", reaction_rates)] + t_remaining = max(0, t_final - t) + checkpoint_cfl = current_cfl + if constant_cfl is True: + dt = ( + current_cfl * get_viscous_timestep(discr=discr, eos=eos, cv=state) + ) + from grudge.op import nodal_min + dt = nodal_min(discr, "vol", dt) + else: + from mirgecom.viscous import get_viscous_cfl + cfl_field = get_viscous_cfl(discr, eos, dt, state) + viz_fields.append(("cfl", cfl_field)) + from grudge.op import nodal_max + checkpoint_cfl = nodal_max(discr, "vol", cfl_field) + dt = min(dt, t_remaining) return sim_checkpoint(discr, visualizer, eos, cv=state, - vizname=casename, step=step, + vizname=casename, step=step, cfl=checkpoint_cfl, t=t, dt=dt, nstatus=nstatus, nviz=nviz, constant_cfl=constant_cfl, comm=comm, viz_fields=viz_fields) @@ -272,8 +261,8 @@ def my_checkpoint(step, t, dt, state): try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=my_checkpoint, - get_timestep=get_timestep, state=current_state, + checkpoint=my_checkpoint, dt=current_dt, + state=current_state, t=current_t, t_final=t_final) except ExactSolutionMismatch as ex: error_state = True diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 47e3b520f..43ed38e52 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -158,7 +158,7 @@ def sim_checkpoint(discr, visualizer, eos, cv, vizname, exact_soln=None, do_viz = check_step(step=step, interval=nviz) do_status = check_step(step=step, interval=nstatus) if do_viz is False and do_status is False: - return 0 + return dt dependent_vars = eos.dependent_vars(cv) @@ -220,6 +220,8 @@ def sim_checkpoint(discr, visualizer, eos, cv, vizname, exact_soln=None, if maxerr > exittol: raise ExactSolutionMismatch(step, t=t, state=cv) + return dt + def check_range_local(discr, dd, field, min_value, max_value): """Check for any negative values.""" diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index ff82ee788..b5e00cff8 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -33,7 +33,8 @@ def _advance_state_stepper_func(rhs, timestepper, checkpoint, get_timestep, - state, t_final, t=0.0, istep=0, logmgr=None, eos=None, dim=None): + state, t_final, dt=0, t=0.0, istep=0, logmgr=None, + eos=None, dim=None): """Advance state from some time (t) to some time (t_final). Parameters @@ -79,14 +80,14 @@ def _advance_state_stepper_func(rhs, timestepper, checkpoint, get_timestep, if logmgr: logmgr.tick_before() + if get_timestep: + dt = get_timestep(state=state) + if checkpoint: + dt = checkpoint(state=state, step=istep, t=t, dt=dt) - dt = get_timestep(state=state) if dt < 0: return istep, t, state - if checkpoint: - checkpoint(state=state, step=istep, t=t, dt=dt) - state = timestepper(state=state, t=t, dt=dt, rhs=rhs) t += dt @@ -212,9 +213,9 @@ def generate_singlerate_leap_advancer(timestepper, component_id, rhs, t, dt, return stepper_cls -def advance_state(rhs, timestepper, checkpoint, get_timestep, state, t_final, - component_id="state", t=0.0, istep=0, logmgr=None, - eos=None, dim=None): +def advance_state(rhs, timestepper, checkpoint, state, t_final, + component_id="state", get_timestep=None, dt=0, t=0.0, + istep=0, logmgr=None, eos=None, dim=None): """Determine what stepper we're using and advance the state from (t) to (t_final). Parameters @@ -281,9 +282,9 @@ def advance_state(rhs, timestepper, checkpoint, get_timestep, state, t_final, else: (current_step, current_t, current_state) = \ _advance_state_stepper_func(rhs=rhs, timestepper=timestepper, - checkpoint=checkpoint, - get_timestep=get_timestep, state=state, - t=t, t_final=t_final, istep=istep, - logmgr=logmgr, eos=eos, dim=dim) + checkpoint=checkpoint, dt=dt, + get_timestep=get_timestep, state=state, + t=t, t_final=t_final, istep=istep, + logmgr=logmgr, eos=eos, dim=dim) return current_step, current_t, current_state diff --git a/mirgecom/viscous.py b/mirgecom/viscous.py index a51dd582b..4a73c252b 100644 --- a/mirgecom/viscous.py +++ b/mirgecom/viscous.py @@ -14,6 +14,7 @@ ^^^^^^^^^^^^^^^^^^^^^ .. autofunction:: get_viscous_timestep +.. autofunction:: get_viscous_cfl """ __copyright__ = """ @@ -221,23 +222,56 @@ def viscous_facial_flux(discr, eos, cv_tpair, grad_cv_tpair, return flux_weak -def get_viscous_timestep(discr, eos, transport_model, cfl, cv): - """Routine (will) return the (local) maximum stable viscous timestep. +def get_viscous_timestep(discr, eos, cv): + """Routine returns the the node-local maximum stable viscous timestep. - Currently, it's a hack waiting for the geometric_factor helpers port - from grudge. + Parameters + ---------- + discr: grudge.eager.EagerDGDiscretization + the discretization to use + eos: mirgecom.eos.GasEOS + An equation of state implementing the speed_of_sound method + cv: :class:`~mirgecom.fluid.ConservedVars` + Fluid solution + + Returns + ------- + class:`~meshmode.dof_array.DOFArray` + The maximum stable timestep at each node. + """ + from grudge.dt_utils import characteristic_lengthscales + from mirgecom.fluid import compute_wavespeed + + length_scales = characteristic_lengthscales(cv.array_context, discr) + + mu = 0 + transport = eos.transport_model() + if transport: + mu = transport.viscosity(eos, cv) + + return( + length_scales / (compute_wavespeed(eos, cv) + mu / length_scales) + ) + + +def get_viscous_cfl(discr, eos, dt, cv): + """Calculate and return node-local CFL based on current state and timestep. + + Parameters + ---------- + discr: :class:`grudge.eager.EagerDGDiscretization` + the discretization to use + eos: mirgecom.eos.GasEOS + Implementing the pressure and temperature functions for + returning pressure and temperature as a function of the state q. + dt: float or :class:`~meshmode.dof_array.DOFArray` + A constant scalar dt or node-local dt + cv: :class:`~mirgecom.fluid.ConservedVars` + The fluid conserved variables + + Returns + ------- + :class:`meshmode.dof_array.DOFArray` + The CFL at each node. """ - dim = discr.dim - mesh = discr.mesh - order = max([grp.order for grp in discr.discr_from_dd("vol").groups]) - nelements = mesh.nelements - nel_1d = nelements ** (1.0 / (1.0 * dim)) - - # This roughly reproduces the timestep AK used in wave toy - dt = (1.0 - 0.25 * (dim - 1)) / (nel_1d * order ** 2) - return cfl * dt -# dt_ngf = dt_non_geometric_factor(discr.mesh) -# dt_gf = dt_geometric_factor(discr.mesh) -# wavespeeds = compute_wavespeed(w,eos=eos) -# max_v = clmath.max(wavespeeds) -# return c*dt_ngf*dt_gf/max_v + return dt / get_viscous_timestep(discr, eos=eos, cv=cv) diff --git a/test/test_viscous.py b/test/test_viscous.py index 1924ff22e..e4a0cfe75 100644 --- a/test/test_viscous.py +++ b/test/test_viscous.py @@ -238,3 +238,60 @@ def test_diffusive_heat_flux(actx_factory): assert discr.norm(j[ispec] - exact_j, np.inf) < tol exact_j = massval * d_alpha[ispec+1] * exact_dy assert discr.norm(j[ispec+1] - exact_j, np.inf) < tol + + +@pytest.mark.parametrize("dim", [1, 2, 3]) +@pytest.mark.parametrize("mu", [-1, 0, 1, 2]) +@pytest.mark.parametrize("vel", [0, 1]) +def test_viscous_timestep(actx_factory, dim, mu, vel): + """Test timestep size.""" + actx = actx_factory() + nel_1d = 5 + + from meshmode.mesh.generation import generate_regular_rect_mesh + + mesh = generate_regular_rect_mesh( + a=(1.0,) * dim, b=(2.0,) * dim, n=(nel_1d,) * dim + ) + + order = 1 + + discr = EagerDGDiscretization(actx, mesh, order=order) + zeros = discr.zeros(actx) + ones = zeros + 1.0 + + velocity = make_obj_array([zeros+vel for _ in range(dim)]) + + massval = 1 + mass = massval*ones + + # I *think* this energy should yield c=1.0 + energy = zeros + 1.0 / (1.4*.4) + mom = mass * velocity + species_mass = None + + cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom, + species_mass=species_mass) + + from grudge.dt_utils import characteristic_lengthscales + chlen = characteristic_lengthscales(actx, discr) + from grudge.op import nodal_min + chlen_min = nodal_min(discr, "vol", chlen) + + mu = mu*chlen_min + if mu < 0: + mu = 0 + tv_model = None + else: + tv_model = SimpleTransport(viscosity=mu) + + eos = IdealSingleGas(transport_model=tv_model) + + from mirgecom.viscous import get_viscous_timestep + dt_field = get_viscous_timestep(discr, eos, cv) + + speed_total = actx.np.sqrt(np.dot(velocity, velocity)) + eos.sound_speed(cv) + dt_expected = chlen / (speed_total + (mu / chlen)) + + error = (dt_expected - dt_field) / dt_expected + assert discr.norm(error, np.inf) == 0 From 41a78f04f7e3c4f10e4bd110e771ba7710b42d64 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 28 Jun 2021 14:29:03 -0500 Subject: [PATCH 195/385] Merge current CFL/DT developments, leap part. --- mirgecom/steppers.py | 38 +++++++++++++++++++++-------------- test/test_time_integrators.py | 2 +- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index b5e00cff8..bbf614217 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -27,14 +27,15 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ - +import numpy as np from logpyle import set_dt from mirgecom.logging_quantities import set_sim_state -def _advance_state_stepper_func(rhs, timestepper, checkpoint, get_timestep, - state, t_final, dt=0, t=0.0, istep=0, logmgr=None, - eos=None, dim=None): +def _advance_state_stepper_func(rhs, timestepper, state, + t_final=np.inf, t=0.0, dt=0, istep=0, + logmgr=None, eos=None, dim=None, checkpoint=None, + get_timestep=None): """Advance state from some time (t) to some time (t_final). Parameters @@ -101,9 +102,10 @@ def _advance_state_stepper_func(rhs, timestepper, checkpoint, get_timestep, return istep, t, state -def _advance_state_leap(rhs, timestepper, checkpoint, get_timestep, - state, t_final, component_id="state", t=0.0, istep=0, - logmgr=None, eos=None, dim=None): +def _advance_state_leap(rhs, timestepper, state, component_id="state", + t_final=np.inf, t=0.0, dt=0.0, istep=0, + logmgr=None, eos=None, dim=None, checkpoint=None, + get_timestep=None): """Advance state from some time *t* to some time *t_final* using :mod:`leap`. Parameters @@ -146,7 +148,9 @@ def _advance_state_leap(rhs, timestepper, checkpoint, get_timestep, return istep, t, state # Generate code for Leap method. - dt = get_timestep(state=state) + if get_timestep: + dt = get_timestep(state=state) + stepper_cls = generate_singlerate_leap_advancer(timestepper, component_id, rhs, t, dt, state) while t < t_final: @@ -154,13 +158,17 @@ def _advance_state_leap(rhs, timestepper, checkpoint, get_timestep, if logmgr: logmgr.tick_before() - dt = get_timestep(state=state) + if get_timestep: + dt = get_timestep(state=state) + + if checkpoint: + dt = checkpoint(state=state, step=istep, t=t, dt=dt) + if dt < 0: return istep, t, state - checkpoint(state=state, step=istep, t=t, dt=dt) - # Leap interface here is *a bit* different. + stepper_cls.dt = dt for event in stepper_cls.run(t_end=t+dt): if isinstance(event, stepper_cls.StateComputed): state = event.state_component @@ -275,10 +283,10 @@ def advance_state(rhs, timestepper, checkpoint, state, t_final, if leap_timestepper: (current_step, current_t, current_state) = \ _advance_state_leap(rhs=rhs, timestepper=timestepper, - checkpoint=checkpoint, - get_timestep=get_timestep, state=state, - t=t, t_final=t_final, component_id=component_id, - istep=istep, logmgr=logmgr, eos=eos, dim=dim) + checkpoint=checkpoint, dt=dt, + get_timestep=get_timestep, state=state, + t=t, t_final=t_final, component_id=component_id, + istep=istep, logmgr=logmgr, eos=eos, dim=dim) else: (current_step, current_t, current_state) = \ _advance_state_stepper_func(rhs=rhs, timestepper=timestepper, diff --git a/test/test_time_integrators.py b/test/test_time_integrators.py index fe69d7514..2a4a96170 100644 --- a/test/test_time_integrators.py +++ b/test/test_time_integrators.py @@ -128,7 +128,7 @@ def get_timestep(state): return dt def my_checkpoint(state, step, t, dt): - return 0 + return dt (step, t, state) = \ advance_state(rhs=rhs, timestepper=method, From 9f1423698aeda20f63d2527a50587e4c840b17b3 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 29 Jun 2021 11:12:43 -0500 Subject: [PATCH 196/385] Attempt to fix pylint issue. --- mirgecom/artificial_viscosity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index b80e34783..553cdb7c7 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -271,7 +271,7 @@ def smoothness_indicator(discr, u, kappa=1.0, s0=-6.0): @memoize_in(actx, (smoothness_indicator, "smooth_comp_knl")) def indicator_prg(): """Compute the smoothness indicator for all elements.""" - from meshmode.array_context import make_loopy_program + from array_context import make_loopy_program return make_loopy_program([ "{[iel]: 0 <= iel < nelements}", "{[idof]: 0 <= idof < ndiscr_nodes_in}", From d8a07fbceac69e392afa75eb3847d9d0a0ca1922 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 29 Jun 2021 11:14:24 -0500 Subject: [PATCH 197/385] Attempt to fix pylint issues. --- mirgecom/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/filter.py b/mirgecom/filter.py index 618350785..2737970b9 100644 --- a/mirgecom/filter.py +++ b/mirgecom/filter.py @@ -146,7 +146,7 @@ def apply_spectral_filter(actx, modal_field, discr, cutoff, DOFArray or object array of DOFArrays """ - from meshmode.array_context import FirstAxisIsElementsTag + from meshmode.transform_metadata import FirstAxisIsElementsTag return DOFArray( actx, tuple(actx.einsum("j,ej->ej", From f445248dc318c7ed7cbd1d30e25d80079f8de219 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 29 Jun 2021 11:33:20 -0500 Subject: [PATCH 198/385] Fix the pylint issue with a proper import. --- mirgecom/artificial_viscosity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 553cdb7c7..33093b7d7 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -271,7 +271,7 @@ def smoothness_indicator(discr, u, kappa=1.0, s0=-6.0): @memoize_in(actx, (smoothness_indicator, "smooth_comp_knl")) def indicator_prg(): """Compute the smoothness indicator for all elements.""" - from array_context import make_loopy_program + from arraycontext import make_loopy_program return make_loopy_program([ "{[iel]: 0 <= iel < nelements}", "{[idof]: 0 <= idof < ndiscr_nodes_in}", From 2ec883faa75edfe6ce88bf92f9d6bde7c8dc019c Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 29 Jun 2021 11:43:40 -0500 Subject: [PATCH 199/385] Remove un-impactful memoizations. --- mirgecom/fluid.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/mirgecom/fluid.py b/mirgecom/fluid.py index b843f2013..8412538f3 100644 --- a/mirgecom/fluid.py +++ b/mirgecom/fluid.py @@ -41,7 +41,6 @@ """ import numpy as np # noqa from pytools.obj_array import make_obj_array -from pytools import memoize_in from meshmode.dof_array import DOFArray # noqa from dataclasses import dataclass, fields from arraycontext import ( @@ -243,10 +242,7 @@ def dim(self): @property def velocity(self): """Return the fluid velocity = momentum / mass.""" - @memoize_in(self, "vel_arry") - def get(): - return self.momentum / self.mass - return get() + return self.momentum / self.mass def join(self): """Call :func:`join_conserved` on *self*.""" @@ -438,9 +434,6 @@ def compute_wavespeed(dim, eos, cv: ConservedVars): where $\mathbf{v}$ is the flow velocity and c is the speed of sound in the fluid. """ - @memoize_in(cv, ("wvspd_arry", type(eos))) - def get(): - actx = cv.array_context - v = cv.velocity - return actx.np.sqrt(np.dot(v, v)) + eos.sound_speed(cv) - return get() + actx = cv.array_context + v = cv.velocity + return actx.np.sqrt(np.dot(v, v)) + eos.sound_speed(cv) From c563de917dbd5762c904303a8f6eee913fa1f27d Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 29 Jun 2021 11:54:18 -0500 Subject: [PATCH 200/385] Remove un-impactful memoizations --- mirgecom/eos.py | 45 ++++++++++++--------------------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/mirgecom/eos.py b/mirgecom/eos.py index 3d6a511ce..142b6017a 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -178,10 +178,7 @@ def pressure(self, cv: ConservedVars): p = (\gamma - 1)e """ - @memoize_in(cv, (IdealSingleGas.pressure, "pressarry")) - def get(): - return self.internal_energy(cv) * (self._gamma - 1.0) - return get() + return self.internal_energy(cv) * (self._gamma - 1.0) def sound_speed(self, cv: ConservedVars): r"""Get the speed of sound in the gas. @@ -192,14 +189,8 @@ def sound_speed(self, cv: ConservedVars): c = \sqrt{\frac{\gamma{p}}{\rho}} """ - actx = cv.mass.array_context - - @memoize_in(cv, (IdealSingleGas.sound_speed, "sosarry")) - def get(): - p = self.pressure(cv) - c2 = self._gamma / cv.mass * p - return actx.np.sqrt(c2) - return get() + actx = cv.array_context + return actx.np.sqrt(self._gamma / cv.mass * self.pressure(cv)) def temperature(self, cv: ConservedVars): r"""Get the thermodynamic temperature of the gas. @@ -212,13 +203,10 @@ def temperature(self, cv: ConservedVars): T = \frac{(\gamma - 1)e}{R\rho} """ - @memoize_in(cv, (IdealSingleGas.temperature, "temparry")) - def get(): - return ( - (((self._gamma - 1.0) / self._gas_const) - * self.internal_energy(cv) / cv.mass) - ) - return get() + return ( + (((self._gamma - 1.0) / self._gas_const) + * self.internal_energy(cv) / cv.mass) + ) def total_energy(self, cv, pressure): r""" @@ -413,9 +401,7 @@ def pressure(self, cv: ConservedVars): def get(): temperature = self.temperature(cv) y = self.species_fractions(cv) - press = self._pyrometheus_mech.get_pressure(cv.mass, temperature, y) - return press - + return self._pyrometheus_mech.get_pressure(cv.mass, temperature, y) return get() def sound_speed(self, cv: ConservedVars): @@ -430,9 +416,8 @@ def sound_speed(self, cv: ConservedVars): @memoize_in(cv, (PyrometheusMixture.sound_speed, type(self._pyrometheus_mech))) def get(): - actx = cv.mass.array_context - c2 = (self.gamma(cv) * self.pressure(cv)) / cv.mass - return actx.np.sqrt(c2) + actx = cv.array_context + return actx.np.sqrt((self.gamma(cv) * self.pressure(cv)) / cv.mass) return get() def temperature(self, cv: ConservedVars): @@ -451,14 +436,8 @@ def temperature(self, cv: ConservedVars): def get(): y = self.species_fractions(cv) e = self.internal_energy(cv) / cv.mass - tmptr = self._pyrometheus_mech.get_temperature(e, self._tguess, - y, True) - # from meshmode.dof_array import freeze - # return freeze(cv.array_context, tmptr) - return tmptr - - # from meshmode.dof_array import thaw - # return thaw(cv.array_context, get()) + return self._pyrometheus_mech.get_temperature(e, self._tguess, + y, True) return get() def total_energy(self, cv, pressure): From 00f3cfa2dbd76d88e72f0cb912f30673c76b7eb3 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 29 Jun 2021 13:15:18 -0500 Subject: [PATCH 201/385] Repair linting issue --- mirgecom/filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/filter.py b/mirgecom/filter.py index 618350785..2737970b9 100644 --- a/mirgecom/filter.py +++ b/mirgecom/filter.py @@ -146,7 +146,7 @@ def apply_spectral_filter(actx, modal_field, discr, cutoff, DOFArray or object array of DOFArrays """ - from meshmode.array_context import FirstAxisIsElementsTag + from meshmode.transform_metadata import FirstAxisIsElementsTag return DOFArray( actx, tuple(actx.einsum("j,ej->ej", From 9f464e07d5a578e7c5efbcdb512230ccb038effa Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 29 Jun 2021 13:26:56 -0500 Subject: [PATCH 202/385] Respond to upstream package imports moving around. #409 --- mirgecom/artificial_viscosity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index b80e34783..33093b7d7 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -271,7 +271,7 @@ def smoothness_indicator(discr, u, kappa=1.0, s0=-6.0): @memoize_in(actx, (smoothness_indicator, "smooth_comp_knl")) def indicator_prg(): """Compute the smoothness indicator for all elements.""" - from meshmode.array_context import make_loopy_program + from arraycontext import make_loopy_program return make_loopy_program([ "{[iel]: 0 <= iel < nelements}", "{[idof]: 0 <= idof < ndiscr_nodes_in}", From 54bdce3b9e6f5c3d76f37a51dcb7300139e3846a Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Tue, 29 Jun 2021 18:14:34 -0700 Subject: [PATCH 203/385] Work around performance issue (#406) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b0654c66a..48fb060b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ PyYAML --editable git+https://github.com/inducer/leap.git#egg=leap --editable git+https://github.com/inducer/modepy.git#egg=modepy --editable git+https://github.com/inducer/arraycontext.git#egg=arraycontext ---editable git+https://github.com/inducer/meshmode.git#egg=meshmode +--editable git+https://github.com/MTCam/meshmode.git@workaround-performance-issue#egg=meshmode --editable git+https://github.com/inducer/grudge.git#egg=grudge --editable git+https://github.com/inducer/pytato.git#egg=pytato --editable git+https://github.com/ecisneros8/pyrometheus.git#egg=pyrometheus From c609760191819c12338570de65f5ae316dba855e Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Wed, 30 Jun 2021 04:20:53 -0700 Subject: [PATCH 204/385] Go back to meshmode main - performance issue resolved. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 48fb060b7..b0654c66a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ PyYAML --editable git+https://github.com/inducer/leap.git#egg=leap --editable git+https://github.com/inducer/modepy.git#egg=modepy --editable git+https://github.com/inducer/arraycontext.git#egg=arraycontext ---editable git+https://github.com/MTCam/meshmode.git@workaround-performance-issue#egg=meshmode +--editable git+https://github.com/inducer/meshmode.git#egg=meshmode --editable git+https://github.com/inducer/grudge.git#egg=grudge --editable git+https://github.com/inducer/pytato.git#egg=pytato --editable git+https://github.com/ecisneros8/pyrometheus.git#egg=pyrometheus From 293d28f180324eeff98615f816ab12cdae3dbc10 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 2 Jul 2021 15:04:51 -0500 Subject: [PATCH 205/385] Refactor autoignit just a bit --- examples/autoignition-mpi.py | 238 ++++++++++++++++++++--------------- mirgecom/steppers.py | 30 ++--- 2 files changed, 151 insertions(+), 117 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 5b97cf7ca..1710fbe7d 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -44,7 +44,6 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( inviscid_sim_timestep, - check_step, generate_and_distribute_mesh, write_visfile ) @@ -74,7 +73,7 @@ @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=False, use_leap=False, use_profiling=False, casename=None, - restart_step=None, restart_name=None): + rst_step=None, rst_name=None): """Drive example.""" cl_ctx = ctx_factory() @@ -86,7 +85,6 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, rank = comm.Get_rank() nproc = comm.Get_size() - restart_file_pattern = "{casename}-{step:04d}-{rank:04d}.pkl" logmgr = initialize_logmgr(use_logmgr, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) @@ -101,54 +99,59 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + # Some discretization parameters dim = 2 nel_1d = 8 order = 1 + # {{{ Time stepping control + # This example runs only 3 steps by default (to keep CI ~short) # With the mixture defined below, equilibrium is achieved at ~40ms # To run to equlibrium, set t_final >= 40ms. + + # Time stepper selection + if use_leap: + from leap.rk import RK4MethodBuilder + timestepper = RK4MethodBuilder("state") + else: + timestepper = rk4_step + + # Time loop control parameters + current_step = 0 t_final = 1e-8 current_cfl = 1.0 - velocity = np.zeros(shape=(dim,)) current_dt = 1e-9 current_t = 0 constant_cfl = False + + # i.o frequencies nstatus = 1 nviz = 5 nhealth = 1 - nlog = 1 nrestart = 5 - rank = 0 - checkpoint_t = current_t - current_step = 0 - if use_leap: - from leap.rk import RK4MethodBuilder - timestepper = RK4MethodBuilder("state") - else: - timestepper = rk4_step - box_ll = -0.005 - box_ur = 0.005 + + # }}} Time stepping control + debug = False - restart_file_pattern = "{casename}-{step:04d}-{rank:04d}.pkl" - restart_path = "restart_data/" - if restart_step: - if not restart_name: - restart_name = casename - rst_filename = ( - restart_path - + restart_file_pattern.format(casename=restart_name, - step=restart_step, rank=rank) - ) + rst_path = "restart_data/" + rst_pattern = ( + rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" + ) + if rst_step: # read the grid from restart data + rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) + from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, rst_filename) + restart_data = read_restart_data(actx, rst_fname) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["nparts"] == nproc - else: + else: # generate the grid from scratch from meshmode.mesh.generation import generate_regular_rect_mesh + box_ll = -0.005 + box_ur = 0.005 generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) local_mesh, global_nelements = generate_and_distribute_mesh(comm, @@ -242,6 +245,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, # Initialize the fluid/gas state with Cantera-consistent data: # (density, pressure, temperature, mass_fractions) print(f"Cantera state (rho,T,P,Y) = ({can_rho}, {can_t}, {can_p}, {can_y}") + velocity = np.zeros(shape=(dim,)) initializer = MixtureInitializer(dim=dim, nspecies=nspecies, pressure=can_p, temperature=can_t, massfractions=can_y, velocity=velocity) @@ -249,9 +253,9 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, my_boundary = AdiabaticSlipBoundary() boundaries = {BTAG_ALL: my_boundary} - if restart_step: + if rst_step: current_t = restart_data["t"] - current_step = restart_step + current_step = rst_step current_state = restart_data["state"] else: # Set the current state from time 0 @@ -294,100 +298,128 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, dt=current_dt, cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) + def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + if do_viz: + my_write_viz(cv, step, t) + if do_restart: + my_write_restart(state=cv, step=step, t=t) + if message is None: + message = "Fatal simulation errors detected." + raise RuntimeError(message) + + def my_write_viz(cv, step, t, dv=None, production_rates=None): + viz_fields = [("cv", cv)] + if dv is not None: + viz_fields.append([("dv", dv)]) + if production_rates is not None: + viz_fields.append([("production_rates", production_rates)]) + write_visfile(discr, viz_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + def my_write_restart(state, step, t): + rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "global_nelements": global_nelements, + "num_parts": nproc + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(dv): + health_error = False + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure) \ + or check_range_local(discr, "vol", dv.pressure, 1e5, 2.4e5): + health_error = True + logger.info(f"{rank=}: Invalid pressure data found.") + + if check_range_local(discr, "vol", dv.temperature, 1.4e3, 3.3e3): + health_error = True + logger.info(f"{rank=}: Invalid temperature data found.") + + return health_error + def my_rhs(t, state): return (euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) + eos.get_species_source_terms(state)) - def post_step_stuff(step, t, dt, state): - do_logend = check_step(step=(step-1), interval=nlog) - - if do_logend and logmgr: + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) logmgr.tick_after() - return state + return state, dt - def my_checkpoint(step, t, dt, state, force=False): - from mirgecom.simutil import check_step - do_viz = force or check_step(step=step, interval=nviz) - do_health = force or check_step(step=step, interval=nhealth) - do_logstart = force or check_step(step=step, interval=nlog) - do_restart = (force or check_step(step, nrestart) - and step != restart_step) - if do_logstart and logmgr: + def my_pre_step(step, t, dt, state): + dv = None + pre_step_errors = False + if logmgr: logmgr.tick_before() - if do_restart: - rst_filename = ( - restart_path - + restart_file_pattern.format(casename=casename, step=step, - rank=rank) - ) - rst_data = { - "local_mesh": local_mesh, - "state": current_state, - "t": t, - "step": step, - "global_nelements": global_nelements, - "num_parts": nproc - } - from mirgecom.restart import write_restart_file - write_restart_file(actx, rst_data, rst_filename, - comm=discr.mpi_communicator) - - if do_viz or do_health: - dv = eos.dependent_vars(state) - reaction_rates = eos.get_production_rates(state) + from mirgecom.simutil import check_step + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) + + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False - errored = False if do_health: - health_message = "" - from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure, 1e5, 2.4e5): - errored = True - health_message += "Invalid pressure data found.\n" - if check_range_local(discr, "vol", dv.temperature, 1.4e3, 3.3e3): - errored = True - health_message += "Temperature data exceeded healthy range.\n" - comm = discr.mpi_communicator + dv = eos.dependent_vars(state) + local_health_error = my_health_check(dv) + health_errors = False if comm is not None: - errored = comm.allreduce(errored, op=MPI.LOR) - if errored: - if rank == 0: - logger.info("Fluid solution failed health check.") - if health_message: # capture any rank's health message - logger.info(f"{rank=}:{health_message}") - - if do_viz or errored: - reaction_rates = eos.get_production_rates(state) - io_fields = [ - ("cv", state), - ("dv", dv), - ("reaction_rates", reaction_rates) - ] - write_visfile(discr, io_fields, visualizer, vizname=casename, - step=step, t=t, overwrite=True) - - if errored: - raise RuntimeError("Error detected by user checkpoint, exiting.") - - return state + health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + if health_errors and rank == 0: + logger.info("Fluid solution failed health check.") + pre_step_errors = pre_step_errors or health_errors + + if do_restart: + my_write_restart(state, step, t) + + if do_viz: + production_rates = eos.get_production_rates(state) + if dv is None: + dv = eos.dependent_vars(state) + my_write_viz(cv=state, dv=dv, step=step, t=t, + production_rates=production_rates) + + if pre_step_errors: + my_graceful_exit(cv=state, step=step, t=t, + do_viz=(not do_viz), do_restart=(not do_restart), + message="Error detected at prestep, exiting.") + + return state, dt current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, - post_step_callback=post_step_stuff, + pre_step_callback=my_pre_step, + post_step_callback=my_post_step, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) - if not check_step(current_step, nviz): # If final step not an output step - if rank == 0: # Then likely something went wrong - logger.info("Checkpointing final state ...") - my_checkpoint(current_step, t=current_t, - dt=(current_t - checkpoint_t), - state=current_state, force=True) + finish_tol = 1e-16 + if np.abs(current_t - t_final) > finish_tol: + my_graceful_exit(cv=current_state, step=current_step, t=current_t, + do_viz=True, do_restart=True, + message="Simulation timestepping did not complete.") + + # Dump the final data + final_dv = eos.dependent_vars(current_state) + final_dm = eos.get_production_rates(current_state) + my_write_viz(cv=current_state, dv=final_dv, production_rates=final_dm, + step=current_step, t=current_t) + my_write_restart(current_state, current_step, current_t) if __name__ == "__main__": diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 3a4413edc..4353d3f6f 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -31,9 +31,10 @@ from mirgecom.logging_quantities import set_sim_state -def _advance_state_stepper_func(rhs, timestepper, get_timestep, - state, t_final, +def _advance_state_stepper_func(rhs, timestepper, + state, t_final, dt=0, t=0.0, istep=0, + get_timestep=None, pre_step_callback=None, post_step_callback=None, logmgr=None, eos=None, dim=None): @@ -87,20 +88,18 @@ def _advance_state_stepper_func(rhs, timestepper, get_timestep, if logmgr: logmgr.tick_before() - dt = get_timestep(state=state) - if dt < 0: - return istep, t, state + if get_timestep: + dt = get_timestep(state=state) if pre_step_callback is not None: - state = pre_step_callback(state=state, step=istep, t=t, dt=dt) + state, dt = pre_step_callback(state=state, step=istep, t=t, dt=dt) state = timestepper(state=state, t=t, dt=dt, rhs=rhs) - t += dt istep += 1 if post_step_callback is not None: - state = post_step_callback(state=state, step=istep, t=t, dt=dt) + state, dt = post_step_callback(state=state, step=istep, t=t, dt=dt) if logmgr: set_dt(logmgr, dt) @@ -111,9 +110,10 @@ def _advance_state_stepper_func(rhs, timestepper, get_timestep, def _advance_state_leap(rhs, timestepper, get_timestep, - state, t_final, + state, t_final, dt=0, component_id="state", t=0.0, istep=0, + get_timestep=None, pre_step_callback=None, post_step_callback=None, logmgr=None, eos=None, dim=None): @@ -163,7 +163,8 @@ def _advance_state_leap(rhs, timestepper, get_timestep, return istep, t, state # Generate code for Leap method. - dt = get_timestep(state=state) + if get_timestep: + dt = get_timestep(state=state) stepper_cls = generate_singlerate_leap_advancer(timestepper, component_id, rhs, t, dt, state) while t < t_final: @@ -232,9 +233,10 @@ def generate_singlerate_leap_advancer(timestepper, component_id, rhs, t, dt, return stepper_cls -def advance_state(rhs, timestepper, get_timestep, state, t_final, +def advance_state(rhs, timestepper, state, t_final, component_id="state", - t=0.0, istep=0, + t=0.0, istep=0, dt=0, + get_timestep=None, pre_step_callback=None, post_step_callback=None, logmgr=None, eos=None, dim=None): @@ -303,7 +305,7 @@ def advance_state(rhs, timestepper, get_timestep, state, t_final, _advance_state_leap( rhs=rhs, timestepper=timestepper, get_timestep=get_timestep, state=state, - t=t, t_final=t_final, + t=t, t_final=t_final, dt=dt, pre_step_callback=pre_step_callback, post_step_callback=post_step_callback, component_id=component_id, @@ -314,7 +316,7 @@ def advance_state(rhs, timestepper, get_timestep, state, t_final, _advance_state_stepper_func( rhs=rhs, timestepper=timestepper, get_timestep=get_timestep, state=state, - t=t, t_final=t_final, + t=t, t_final=t_final, dt=dt, pre_step_callback=pre_step_callback, post_step_callback=post_step_callback, istep=istep, From 8443a66a2ef23c66a0077210ca2c5181606c98db Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 3 Jul 2021 07:58:21 -0500 Subject: [PATCH 206/385] Refactor autoignition example per discussions. --- examples/autoignition-mpi.py | 5 +++-- mirgecom/steppers.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 1710fbe7d..ca9a3d91d 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -312,9 +312,9 @@ def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): def my_write_viz(cv, step, t, dv=None, production_rates=None): viz_fields = [("cv", cv)] if dv is not None: - viz_fields.append([("dv", dv)]) + viz_fields.append(("dv", dv)) if production_rates is not None: - viz_fields.append([("production_rates", production_rates)]) + viz_fields.append(("production_rates", production_rates)) write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) @@ -362,6 +362,7 @@ def my_post_step(step, t, dt, state): def my_pre_step(step, t, dt, state): dv = None pre_step_errors = False + if logmgr: logmgr.tick_before() diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 4353d3f6f..622746818 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -109,8 +109,8 @@ def _advance_state_stepper_func(rhs, timestepper, return istep, t, state -def _advance_state_leap(rhs, timestepper, get_timestep, - state, t_final, dt=0, +def _advance_state_leap(rhs, timestepper, state, + t_final, dt=0, component_id="state", t=0.0, istep=0, get_timestep=None, From d2b2c683b4251bda2820dbc721cb267df94ef46a Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 3 Jul 2021 08:03:35 -0500 Subject: [PATCH 207/385] Bring examples up-to-date with current stepper API --- examples/lump-mpi.py | 2 +- examples/mixture-mpi.py | 2 +- examples/pulse-mpi.py | 2 +- examples/scalar-lump-mpi.py | 2 +- examples/sod-mpi.py | 2 +- examples/vortex-mpi.py | 12 ++++-------- 6 files changed, 9 insertions(+), 13 deletions(-) diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 8ca64e5d2..99e285074 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -184,7 +184,7 @@ def my_checkpoint(step, t, dt, state): if errored: raise RuntimeError("Error detected by user checkpoint, exiting.") - return state + return state, dt current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index ca20bce37..6f4dcf470 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -202,7 +202,7 @@ def my_checkpoint(step, t, dt, state): if errored: raise RuntimeError("Error detected by user checkpoint, exiting.") - return state + return state, dt current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index b10aafa7a..8e761f826 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -195,7 +195,7 @@ def my_checkpoint(step, t, dt, state): if errored: raise RuntimeError("Error detected by user checkpoint, exiting.") - return state + return state, dt current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index a434835b1..4cf40dd8b 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -193,7 +193,7 @@ def my_checkpoint(step, t, dt, state): if errored: raise RuntimeError("Error detected by user checkpoint, exiting.") - return state + return state, dt current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 4e4526853..119f8d309 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -182,7 +182,7 @@ def my_checkpoint(step, t, dt, state): if errored: raise RuntimeError("Error detected by user checkpoint, exiting.") - return state + return state, dt current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 04f5603f8..3a70d7716 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -107,8 +107,6 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal nstatus = 10 nviz = 10 nhealth = 10 - nlog = 10 - rank = 0 checkpoint_t = current_t current_step = 0 if use_leap: @@ -187,20 +185,18 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) def post_step_stuff(step, t, dt, state): - do_log = check_step(step=(step-1), interval=nlog) - if do_log and logmgr: + if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) logmgr.tick_after() - return state + return state, dt def my_checkpoint(step, t, dt, state): do_status = check_step(step=step, interval=nstatus) do_viz = check_step(step=step, interval=nviz) do_health = check_step(step=step, interval=nhealth) - do_log = check_step(step=step, interval=nlog) - if do_log and logmgr: + if logmgr: logmgr.tick_before() if do_status or do_viz or do_health: @@ -247,7 +243,7 @@ def my_checkpoint(step, t, dt, state): if errored: raise RuntimeError("Error detected by user checkpoint, exiting.") - return state + return state, dt current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, From 9a5fcf3d3ec6de2832557db883ee2e098869f738 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 3 Jul 2021 08:14:15 -0500 Subject: [PATCH 208/385] Add DT health check to autoignition. --- examples/autoignition-mpi.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index ca9a3d91d..fa03010b6 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -331,7 +331,7 @@ def my_write_restart(state, step, t): from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) - def my_health_check(dv): + def my_health_check(dv, dt): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ @@ -343,6 +343,11 @@ def my_health_check(dv): health_error = True logger.info(f"{rank=}: Invalid temperature data found.") + if dt < 0: + health_error = True + if rank == 0: + logger.info("Global DT is negative!") + return health_error def my_rhs(t, state): @@ -377,7 +382,7 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - local_health_error = my_health_check(dv) + local_health_error = my_health_check(dv, dt) health_errors = False if comm is not None: health_errors = comm.allreduce(local_health_error, op=MPI.LOR) From de48784f5d5b49f56083d8bf9d6d2ddee30206ba Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 6 Jul 2021 16:38:44 -0500 Subject: [PATCH 209/385] Modernize lump example --- examples/lump-mpi.py | 244 ++++++++++++++++++++++++++++++++----------- 1 file changed, 183 insertions(+), 61 deletions(-) diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 99e285074..978696e7b 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -50,16 +50,47 @@ from mirgecom.initializers import Lump from mirgecom.eos import IdealSingleGas +from logpyle import IntervalTimer, set_dt +from mirgecom.euler import extract_vars_for_logging, units_for_logging +from mirgecom.profiling import PyOpenCLProfilingArrayContext +from mirgecom.logging_quantities import ( + initialize_logmgr, + logmgr_add_many_discretization_quantities, + logmgr_add_device_name, + logmgr_add_device_memory_usage, + set_sim_state +) logger = logging.getLogger(__name__) @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False): +def main(ctx_factory=cl.create_some_context, use_leap=False, + use_profiling=False, rst_step=None, rst_name=None, + casename="lump", use_logmgr=True): """Drive example.""" cl_ctx = ctx_factory() - queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, + + if casename is None: + casename = "mirgecom" + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + nparts = comm.Get_size() + + logmgr = initialize_logmgr(use_logmgr, + filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) + + if use_profiling: + queue = cl.CommandQueue(cl_ctx, + properties=cl.command_queue_properties.PROFILING_ENABLE) + actx = PyOpenCLProfilingArrayContext(queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), + logmgr=logmgr) + else: + queue = cl.CommandQueue(cl_ctx) + actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) dim = 3 @@ -74,14 +105,13 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): current_t = 0 eos = IdealSingleGas() initializer = Lump(dim=dim, center=orig, velocity=vel) - casename = "lump" boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} constant_cfl = False nstatus = 1 nhealth = 1 + nrestart = 10 nviz = 1 rank = 0 - checkpoint_t = current_t current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder @@ -91,20 +121,50 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): box_ll = -5.0 box_ur = 5.0 - from mpi4py import MPI - comm = MPI.COMM_WORLD - rank = comm.Get_rank() + rst_path = "restart_data/" + rst_pattern = ( + rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" + ) + if rst_step: # read the grid from restart data + rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) - from meshmode.mesh.generation import generate_regular_rect_mesh - generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,) * dim, - b=(box_ur,) * dim, nelements_per_axis=(nel_1d,) * dim) - local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) - local_nelements = local_mesh.nelements + from mirgecom.restart import read_restart_data + restart_data = read_restart_data(actx, rst_fname) + local_mesh = restart_data["local_mesh"] + local_nelements = local_mesh.nelements + global_nelements = restart_data["global_nelements"] + assert restart_data["nparts"] == nparts + else: # generate the grid from scratch + from meshmode.mesh.generation import generate_regular_rect_mesh + generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, + b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) + local_mesh, global_nelements = generate_and_distribute_mesh(comm, + generate_mesh) + local_nelements = local_mesh.nelements discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) nodes = thaw(actx, discr.nodes()) + + if logmgr: + logmgr_add_device_name(logmgr, queue) + logmgr_add_device_memory_usage(logmgr, queue) + logmgr_add_many_discretization_quantities(logmgr, discr, dim, + extract_vars_for_logging, units_for_logging) + + logmgr.add_watches([ + ("step.max", "step = {value}, "), + ("t_sim.max", "sim time: {value:1.6e} s\n"), + ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), + ("max_pressure", "{value:1.9e})\n"), + ("t_step.max", "------- step walltime: {value:6g} s, "), + ("t_log.max", "log walltime: {value:6g} s") + ]) + + vis_timer = IntervalTimer("t_vis", "Time spent visualizing") + logmgr.add_quantity(vis_timer) + current_state = initializer(nodes) visualizer = make_visualizer(discr) @@ -124,80 +184,142 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): dt=current_dt, cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) + def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + if do_viz: + my_write_viz(cv, step, t) + if do_restart: + my_write_restart(state=cv, step=step, t=t) + if message is None: + message = "Fatal simulation errors detected." + raise RuntimeError(message) + + def my_write_viz(cv, step, t, dv=None, exact_mix=None, resid=None): + viz_fields = [("cv", cv)] + if dv is not None: + viz_fields.append(("dv", dv)) + if exact_mix is not None: + viz_fields.append(("exact", exact_mix)) + if resid is not None: + viz_fields.append(("residual", resid)) + from mirgecom.simutil import write_visfile + write_visfile(discr, viz_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + def my_write_restart(state, step, t): + rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "global_nelements": global_nelements, + "num_parts": nparts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(state, dv, exact_mix): + health_error = False + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure) \ + or check_range_local(discr, "vol", dv.pressure, .9, 1.1): + health_error = True + logger.info(f"{rank=}: Invalid pressure data found.") + + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact_mix) + exittol = .09 + if max(component_errors) > exittol: + health_error = True + if rank == 0: + logger.info("Solution diverged from exact_mix.") + + return health_error + def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) - def my_checkpoint(step, t, dt, state): + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + + def my_pre_step(step, t, dt, state): + dv = None + exact_mix = None + pre_step_errors = False + + if logmgr: + logmgr.tick_before() + from mirgecom.simutil import check_step - do_status = check_step(step=step, interval=nstatus) do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) + do_status = check_step(step=step, interval=nstatus) - if do_status or do_viz or do_health: - from mirgecom.simutil import compare_fluid_solutions + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False + + if do_health: dv = eos.dependent_vars(state) exact_mix = initializer(x_vec=nodes, eos=eos, t=t) - component_errors = compare_fluid_solutions(discr, state, exact_mix) + local_health_error = my_health_check(state, dv, exact_mix) + health_errors = False + if comm is not None: + health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + if health_errors and rank == 0: + logger.info("Fluid solution failed health check.") + pre_step_errors = pre_step_errors or health_errors + + if do_restart: + my_write_restart(state, step, t) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + if exact_mix is None: + exact_mix = initializer(x_vec=nodes, eos=eos, t=t) resid = state - exact_mix - io_fields = [ - ("cv", state), - ("dv", dv), - ("exact_mix", exact_mix), - ("resid", resid) - ] - - if do_status: # This is bad, logging already completely replaces this - from mirgecom.io import make_status_message - status_msg = make_status_message(discr=discr, t=t, step=step, dt=dt, - cfl=current_cfl, dependent_vars=dv) - status_msg += ( - "\n------- errors=" + my_write_viz(cv=state, dv=dv, step=step, t=t, + exact_mix=exact_mix, resid=resid) + + if do_status: + if exact_mix is None: + exact_mix = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact_mix) + status_msg = ( + "------- errors=" + ", ".join("%.3g" % en for en in component_errors)) if rank == 0: logger.info(status_msg) - errored = False - if do_health: - from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure, .9, 1.1): - errored = True - message = "Invalid pressure data found.\n" - exittol = .09 - if max(component_errors) > exittol: - errored = True - message += "Solution diverged from exact_mix.\n" - comm = discr.mpi_communicator - if comm is not None: - errored = comm.allreduce(errored, op=MPI.LOR) - if errored: - if rank == 0: - logger.info("Fluid solution failed health check.") - logger.info(message) # do this on all ranks - - if do_viz or errored > 0: - from mirgecom.simutil import write_visfile - write_visfile(discr, io_fields, visualizer, vizname=casename, - step=step, t=t, overwrite=True) - - if errored: - raise RuntimeError("Error detected by user checkpoint, exiting.") + if pre_step_errors: + my_graceful_exit(cv=state, step=step, t=t, + do_viz=(not do_viz), do_restart=(not do_restart), + message="Error detected at prestep, exiting.") return state, dt current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, + pre_step_callback=my_pre_step, + post_step_callback=my_post_step, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) # if current_t != checkpoint_t: if rank == 0: logger.info("Checkpointing final state ...") - my_checkpoint(current_step, t=current_t, - dt=(current_t - checkpoint_t), - state=current_state) if __name__ == "__main__": From a5e5fc4271de43cfac601690e39dddbd1e618d76 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 6 Jul 2021 16:39:17 -0500 Subject: [PATCH 210/385] remove unused exceptions module, for now --- mirgecom/exceptions.py | 45 ------------------------------------------ 1 file changed, 45 deletions(-) delete mode 100644 mirgecom/exceptions.py diff --git a/mirgecom/exceptions.py b/mirgecom/exceptions.py deleted file mode 100644 index 8d43c81a1..000000000 --- a/mirgecom/exceptions.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Provide custom exceptions for use in callback routines. - -.. autoexception:: SynchronizedError -.. autoexception:: SimulationHealthError -""" - -__copyright__ = """ -Copyright (C) 2021 University of Illinois Board of Trustees -""" - -__license__ = """ -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -""" - - -class SynchronizedError(Exception): - """Exception base class which must be globally synchronized. - - .. attribute:: message - - A :class:`str` describing the message for the global exception. - """ - - def __init__(self, message): - super().__init__(message) - - -class SimulationHealthError(SynchronizedError): - """Exception class for an unphysical simulation.""" From 42ea719b034ce138598df886e960947f6cc0bc6c Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 6 Jul 2021 16:39:37 -0500 Subject: [PATCH 211/385] remove unused exceptions module, for now --- doc/support/tools.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/support/tools.rst b/doc/support/tools.rst index f8e0b7c07..d29d63c2e 100644 --- a/doc/support/tools.rst +++ b/doc/support/tools.rst @@ -1,6 +1,5 @@ Random Pile'o'Tools =================== -.. automodule:: mirgecom.exceptions .. automodule:: mirgecom.simutil .. automodule:: mirgecom.utils From 0d74f9c9f10aa49d0c0290310fcc2a97b9b2cf44 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 6 Jul 2021 16:40:07 -0500 Subject: [PATCH 212/385] fix api/signature and doc --- mirgecom/steppers.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 622746818..c0e283338 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -65,11 +65,11 @@ def _advance_state_stepper_func(rhs, timestepper, Step number from which to start pre_step_callback An optional user-defined function, with signature: - ``state = pre_step_callback(state, step, t, dt)``, + ``state, dt = pre_step_callback(state, step, t, dt)``, to be called before the timestepper is called for that particular step. post_step_callback An optional user-defined function, with signature: - ``state = post_step_callback(state, step, t, dt)``, + ``state, dt = post_step_callback(state, step, t, dt)``, to be called after the timestepper is called for that particular step. Returns @@ -144,11 +144,11 @@ def _advance_state_leap(rhs, timestepper, state, Step number from which to start pre_step_callback An optional user-defined function, with signature: - ``state = pre_step_callback(state, step, t, dt)``, + ``state, dt = pre_step_callback(state, step, t, dt)``, to be called before the timestepper is called for that particular step. post_step_callback An optional user-defined function, with signature: - ``state = post_step_callback(state, step, t, dt)``, + ``state, dt = post_step_callback(state, step, t, dt)``, to be called after the timestepper is called for that particular step. Returns @@ -174,9 +174,9 @@ def _advance_state_leap(rhs, timestepper, state, return istep, t, state if pre_step_callback is not None: - state = pre_step_callback(state=state, - step=istep, - t=t, dt=dt) + state, dt = pre_step_callback(state=state, + step=istep, + t=t, dt=dt) # Leap interface here is *a bit* different. for event in stepper_cls.run(t_end=t+dt): @@ -185,9 +185,9 @@ def _advance_state_leap(rhs, timestepper, state, t += dt if post_step_callback is not None: - state = post_step_callback(state=state, - step=istep, - t=t, dt=dt) + state, dt = post_step_callback(state=state, + step=istep, + t=t, dt=dt) istep += 1 @@ -273,11 +273,11 @@ def advance_state(rhs, timestepper, state, t_final, Step number from which to start pre_step_callback An optional user-defined function, with signature: - ``state = pre_step_callback(state, step, t, dt)``, + ``state, dt = pre_step_callback(state, step, t, dt)``, to be called before the timestepper is called for that particular step. post_step_callback An optional user-defined function, with signature: - ``state = post_step_callback(state, step, t, dt)``, + ``state, dt = post_step_callback(state, step, t, dt)``, to be called after the timestepper is called for that particular step. Returns From ae90d9f367f17c9d15e835e6295477f5ca5f1a4a Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 6 Jul 2021 19:15:14 -0500 Subject: [PATCH 213/385] Update stepper API to current --- mirgecom/steppers.py | 4 +++- test/test_time_integrators.py | 14 +++----------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index c0e283338..9eef90a68 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -169,7 +169,9 @@ def _advance_state_leap(rhs, timestepper, state, rhs, t, dt, state) while t < t_final: - dt = get_timestep(state=state) + if get_timestep: + dt = get_timestep(state=state) + if dt < 0: return istep, t, state diff --git a/test/test_time_integrators.py b/test/test_time_integrators.py index cc55ceeca..3660b0061 100644 --- a/test/test_time_integrators.py +++ b/test/test_time_integrators.py @@ -123,18 +123,10 @@ def rhs(t, y): t_final = 4 step = 0 - # Callables needed for advance_state and advance_state_leap - def get_timestep(state): - return dt - - def my_checkpoint(state, step, t, dt): - return state - (step, t, state) = \ - advance_state(rhs=rhs, timestepper=method, - get_timestep=get_timestep, state=state, - t=t, t_final=t_final, component_id="y", - post_step_callback=my_checkpoint) + advance_state(rhs=rhs, timestepper=method, dt=dt, + state=state, t=t, t_final=t_final, + component_id="y") error = np.abs(state - exact_soln(t)) / exact_soln(t) integrator_eoc.add_data_point(dt, error) From d092b82ef12e60e65cbb3e65b8a0a2e653beaf6a Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 6 Jul 2021 19:57:10 -0500 Subject: [PATCH 214/385] Modernize more examples. --- examples/autoignition-mpi.py | 2 +- examples/lump-mpi.py | 55 ++++--- examples/mixture-mpi.py | 274 ++++++++++++++++++++++++++--------- examples/pulse-mpi.py | 238 ++++++++++++++++++++++-------- 4 files changed, 420 insertions(+), 149 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index fa03010b6..e0ce7429d 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -72,7 +72,7 @@ @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=False, - use_leap=False, use_profiling=False, casename=None, + use_leap=False, use_profiling=False, casename="autoignition", rst_step=None, rst_name=None): """Drive example.""" cl_ctx = ctx_factory() diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 978696e7b..079c4e47f 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -165,7 +165,13 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) - current_state = initializer(nodes) + if rst_step: + current_t = restart_data["t"] + current_step = rst_step + current_state = restart_data["state"] + else: + # Set the current state from time 0 + current_state = initializer(nodes) visualizer = make_visualizer(discr) initname = initializer.__class__.__name__ @@ -195,12 +201,12 @@ def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): message = "Fatal simulation errors detected." raise RuntimeError(message) - def my_write_viz(cv, step, t, dv=None, exact_mix=None, resid=None): + def my_write_viz(cv, step, t, dv=None, exact_lump=None, resid=None): viz_fields = [("cv", cv)] if dv is not None: viz_fields.append(("dv", dv)) - if exact_mix is not None: - viz_fields.append(("exact", exact_mix)) + if exact_lump is not None: + viz_fields.append(("exact", exact_lump)) if resid is not None: viz_fields.append(("residual", resid)) from mirgecom.simutil import write_visfile @@ -220,7 +226,7 @@ def my_write_restart(state, step, t): from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) - def my_health_check(state, dv, exact_mix): + def my_health_check(state, dv, exact_lump): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ @@ -229,12 +235,12 @@ def my_health_check(state, dv, exact_mix): logger.info(f"{rank=}: Invalid pressure data found.") from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact_mix) + component_errors = compare_fluid_solutions(discr, state, exact_lump) exittol = .09 if max(component_errors) > exittol: health_error = True if rank == 0: - logger.info("Solution diverged from exact_mix.") + logger.info("Solution diverged from exact_lump.") return health_error @@ -253,7 +259,7 @@ def my_post_step(step, t, dt, state): def my_pre_step(step, t, dt, state): dv = None - exact_mix = None + exact_lump = None pre_step_errors = False if logmgr: @@ -271,8 +277,8 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - exact_mix = initializer(x_vec=nodes, eos=eos, t=t) - local_health_error = my_health_check(state, dv, exact_mix) + exact_lump = initializer(x_vec=nodes, eos=eos, t=t) + local_health_error = my_health_check(state, dv, exact_lump) health_errors = False if comm is not None: health_errors = comm.allreduce(local_health_error, op=MPI.LOR) @@ -286,17 +292,17 @@ def my_pre_step(step, t, dt, state): if do_viz: if dv is None: dv = eos.dependent_vars(state) - if exact_mix is None: - exact_mix = initializer(x_vec=nodes, eos=eos, t=t) - resid = state - exact_mix + if exact_lump is None: + exact_lump = initializer(x_vec=nodes, eos=eos, t=t) + resid = state - exact_lump my_write_viz(cv=state, dv=dv, step=step, t=t, - exact_mix=exact_mix, resid=resid) + exact_lump=exact_lump, resid=resid) if do_status: - if exact_mix is None: - exact_mix = initializer(x_vec=nodes, eos=eos, t=t) + if exact_lump is None: + exact_lump = initializer(x_vec=nodes, eos=eos, t=t) from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact_mix) + component_errors = compare_fluid_solutions(discr, state, exact_lump) status_msg = ( "------- errors=" + ", ".join("%.3g" % en for en in component_errors)) @@ -317,10 +323,23 @@ def my_pre_step(step, t, dt, state): get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) - # if current_t != checkpoint_t: + finish_tol = 1e-16 + if np.abs(current_t - t_final) > finish_tol: + my_graceful_exit(cv=current_state, step=current_step, t=current_t, + do_viz=True, do_restart=True, + message="Simulation timestepping did not complete.") + + # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") + final_dv = eos.dependent_vars(current_state) + final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) + final_resid = current_state - final_exact + my_write_viz(cv=current_state, dv=final_dv, exact_lump=final_exact, + resid=final_resid, step=current_step, t=current_t) + my_write_restart(current_state, current_step, current_t) + if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 6f4dcf470..7bb3e6993 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -53,21 +53,52 @@ import cantera import pyrometheus as pyro +from logpyle import IntervalTimer, set_dt +from mirgecom.euler import extract_vars_for_logging, units_for_logging +from mirgecom.profiling import PyOpenCLProfilingArrayContext +from mirgecom.logging_quantities import ( + initialize_logmgr, + logmgr_add_many_discretization_quantities, + logmgr_add_device_name, + logmgr_add_device_memory_usage, + set_sim_state +) + logger = logging.getLogger(__name__) @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False): +def main(ctx_factory=cl.create_some_context, use_leap=False, + use_profiling=False, rst_step=None, rst_name=None, + casename="uiuc_mixture", use_logmgr=True): """Drive example.""" cl_ctx = ctx_factory() - queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, + + if casename is None: + casename = "mirgecom" + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + nparts = comm.Get_size() + + logmgr = initialize_logmgr(use_logmgr, + filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) + + if use_profiling: + queue = cl.CommandQueue(cl_ctx, + properties=cl.command_queue_properties.PROFILING_ENABLE) + actx = PyOpenCLProfilingArrayContext(queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), + logmgr=logmgr) + else: + queue = cl.CommandQueue(cl_ctx) + actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) dim = 3 nel_1d = 16 order = 3 - exittol = 10.0 t_final = 0.002 current_cfl = 1.0 velocity = np.zeros(shape=(dim,)) @@ -77,33 +108,61 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): constant_cfl = False nstatus = 1 nhealth = 1 + nrestart = 5 nviz = 1 rank = 0 - checkpoint_t = current_t current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step - box_ll = -5.0 - box_ur = 5.0 - from mpi4py import MPI - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - - from meshmode.mesh.generation import generate_regular_rect_mesh - generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,) * dim, - b=(box_ur,) * dim, nelements_per_axis=(nel_1d,) * dim) - local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) - local_nelements = local_mesh.nelements + rst_path = "restart_data/" + rst_pattern = ( + rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" + ) + if rst_step: # read the grid from restart data + rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) + + from mirgecom.restart import read_restart_data + restart_data = read_restart_data(actx, rst_fname) + local_mesh = restart_data["local_mesh"] + local_nelements = local_mesh.nelements + global_nelements = restart_data["global_nelements"] + assert restart_data["nparts"] == nparts + else: # generate the grid from scratch + box_ll = -5.0 + box_ur = 5.0 + from meshmode.mesh.generation import generate_regular_rect_mesh + generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, + b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) + local_mesh, global_nelements = generate_and_distribute_mesh(comm, + generate_mesh) + local_nelements = local_mesh.nelements discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) nodes = thaw(actx, discr.nodes()) - casename = "uiuc_mixture" + + if logmgr: + logmgr_add_device_name(logmgr, queue) + logmgr_add_device_memory_usage(logmgr, queue) + logmgr_add_many_discretization_quantities(logmgr, discr, dim, + extract_vars_for_logging, units_for_logging) + + logmgr.add_watches([ + ("step.max", "step = {value}, "), + ("t_sim.max", "sim time: {value:1.6e} s\n"), + ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), + ("max_pressure", "{value:1.9e})\n"), + ("t_step.max", "------- step walltime: {value:6g} s, "), + ("t_log.max", "log walltime: {value:6g} s") + ]) + + vis_timer = IntervalTimer("t_vis", "Time spent visualizing") + logmgr.add_quantity(vis_timer) # Pyrometheus initialization from mirgecom.mechanisms import get_mechanism_cti @@ -126,7 +185,13 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} nodes = thaw(actx, discr.nodes()) - current_state = initializer(x_vec=nodes, eos=eos) + if rst_step: + current_t = restart_data["t"] + current_step = rst_step + current_state = restart_data["state"] + else: + # Set the current state from time 0 + current_state = initializer(x_vec=nodes, eos=eos) visualizer = make_visualizer(discr) initname = initializer.__class__.__name__ @@ -145,77 +210,154 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): dt=current_dt, cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) + def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + if do_viz: + my_write_viz(cv, step, t) + if do_restart: + my_write_restart(state=cv, step=step, t=t) + if message is None: + message = "Fatal simulation errors detected." + raise RuntimeError(message) + + def my_write_viz(cv, step, t, dv=None, exact_mix=None, resid=None): + viz_fields = [("cv", cv)] + if dv is not None: + viz_fields.append(("dv", dv)) + if exact_mix is not None: + viz_fields.append(("exact", exact_mix)) + if resid is not None: + viz_fields.append(("residual", resid)) + from mirgecom.simutil import write_visfile + write_visfile(discr, viz_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + def my_write_restart(state, step, t): + rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "global_nelements": global_nelements, + "num_parts": nparts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(state, dv, exact_mix): + health_error = False + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure) \ + or check_range_local(discr, "vol", dv.pressure, 1e5, 1.1e5): + health_error = True + logger.info(f"{rank=}: Invalid pressure data found.") + + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact_mix) + exittol = .09 + if max(component_errors) > exittol: + health_error = True + if rank == 0: + logger.info("Solution diverged from exact_mix.") + + return health_error + def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) - def my_checkpoint(step, t, dt, state): + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + + def my_pre_step(step, t, dt, state): + dv = None + exact_mix = None + pre_step_errors = False + + if logmgr: + logmgr.tick_before() + from mirgecom.simutil import check_step - do_status = check_step(step=step, interval=nstatus) do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) + do_status = check_step(step=step, interval=nstatus) - if do_status or do_viz or do_health: - from mirgecom.simutil import compare_fluid_solutions + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False + + if do_health: dv = eos.dependent_vars(state) exact_mix = initializer(x_vec=nodes, eos=eos, t=t) - component_errors = compare_fluid_solutions(discr, state, exact_mix) + local_health_error = my_health_check(state, dv, exact_mix) + health_errors = False + if comm is not None: + health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + if health_errors and rank == 0: + logger.info("Fluid solution failed health check.") + pre_step_errors = pre_step_errors or health_errors + + if do_restart: + my_write_restart(state, step, t) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + if exact_mix is None: + exact_mix = initializer(x_vec=nodes, eos=eos, t=t) resid = state - exact_mix - io_fields = [ - ("cv", state), - ("dv", dv), - ("exact_mix", exact_mix), - ("resid", resid) - ] - - if do_status: # This is bad, logging already completely replaces this - from mirgecom.io import make_status_message - status_msg = make_status_message(discr=discr, t=t, step=step, dt=dt, - cfl=current_cfl, dependent_vars=dv) - status_msg += ( - "\n------- errors=" + my_write_viz(cv=state, dv=dv, step=step, t=t, + exact_mix=exact_mix, resid=resid) + + if do_status: + if exact_mix is None: + exact_mix = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact_mix) + status_msg = ( + "------- errors=" + ", ".join("%.3g" % en for en in component_errors)) if rank == 0: logger.info(status_msg) - errored = False - if do_health: - from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure, 1e5, 1.1e5): - errored = True - message = "Invalid pressure data found.\n" - if max(component_errors) > exittol: - errored = True - message += "Solution diverged from exact_mix.\n" - errored = discr.mpi_communicator.allreduce(errored, op=MPI.LOR) - if errored: - if rank == 0: - logger.info("Fluid solution failed health check.") - logger.info(message) # do this on all ranks - - if do_viz or errored: - from mirgecom.simutil import write_visfile - write_visfile(discr, io_fields, visualizer, vizname=casename, - step=step, t=t, overwrite=True) - - if errored: - raise RuntimeError("Error detected by user checkpoint, exiting.") + if pre_step_errors: + my_graceful_exit(cv=state, step=step, t=t, + do_viz=(not do_viz), do_restart=(not do_restart), + message="Error detected at prestep, exiting.") return state, dt current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, + pre_step_callback=my_pre_step, + post_step_callback=my_post_step, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) - if current_t != checkpoint_t: # This check because !overwrite - if rank == 0: - logger.info("Checkpointing final state ...") - my_checkpoint(current_step, t=current_t, - dt=(current_t - checkpoint_t), - state=current_state) + finish_tol = 1e-16 + if np.abs(current_t - t_final) > finish_tol: + my_graceful_exit(cv=current_state, step=current_step, t=current_t, + do_viz=True, do_restart=True, + message="Simulation timestepping did not complete.") + + # Dump the final data + final_dv = eos.dependent_vars(current_state) + final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) + final_resid = current_state - final_exact + my_write_viz(cv=current_state, dv=final_dv, exact_mix=final_exact, + resid=final_resid, step=current_step, t=current_t) + my_write_restart(current_state, current_step, current_t) + if rank == 0: + logger.info("Checkpointing final state ...") if __name__ == "__main__": diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 8e761f826..192315ccf 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -56,16 +56,53 @@ ) from mirgecom.eos import IdealSingleGas +from logpyle import set_dt +from mirgecom.euler import extract_vars_for_logging, units_for_logging +from mirgecom.profiling import PyOpenCLProfilingArrayContext +from mirgecom.logging_quantities import ( + initialize_logmgr, + logmgr_add_many_discretization_quantities, + logmgr_add_device_name, + logmgr_add_device_memory_usage, + set_sim_state +) + +logger = logging.getLogger(__name__) + @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False): +def main(ctx_factory=cl.create_some_context, use_logmgr=False, + use_leap=False, use_profiling=False, casename="pulse", + rst_step=None, rst_name=None): """Drive the example.""" cl_ctx = ctx_factory() queue = cl.CommandQueue(cl_ctx) actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - logger = logging.getLogger(__name__) + cl_ctx = ctx_factory() + + if casename is None: + casename = "mirgecom" + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + num_parts = comm.Get_size() + + logmgr = initialize_logmgr(use_logmgr, + filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) + + if use_profiling: + queue = cl.CommandQueue(cl_ctx, + properties=cl.command_queue_properties.PROFILING_ENABLE) + actx = PyOpenCLProfilingArrayContext(queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), + logmgr=logmgr) + else: + queue = cl.CommandQueue(cl_ctx) + actx = PyOpenCLArrayContext(queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) dim = 2 nel_1d = 16 @@ -85,47 +122,70 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): boundaries = {BTAG_ALL: wall} constant_cfl = False nstatus = 1 + nrestart = 5 nviz = 10 nhealth = 1 rank = 0 - checkpoint_t = current_t current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step - box_ll = -0.5 - box_ur = 0.5 - - from mpi4py import MPI - comm = MPI.COMM_WORLD - nproc = comm.Get_size() - rank = comm.Get_rank() - num_parts = nproc - from meshmode.mesh.generation import generate_regular_rect_mesh - if num_parts > 1: - generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,) * dim, - b=(box_ur,) * dim, - nelements_per_axis=(nel_1d,) * dim) + rst_path = "restart_data/" + rst_pattern = ( + rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" + ) + if rst_step: # read the grid from restart data + rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) + + from mirgecom.restart import read_restart_data + restart_data = read_restart_data(actx, rst_fname) + local_mesh = restart_data["local_mesh"] + local_nelements = local_mesh.nelements + global_nelements = restart_data["global_nelements"] + assert restart_data["nparts"] == num_parts + else: # generate the grid from scratch + from meshmode.mesh.generation import generate_regular_rect_mesh + box_ll = -0.5 + box_ur = 0.5 + generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, + b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) - else: - local_mesh = generate_regular_rect_mesh( - a=(box_ll,) * dim, b=(box_ur,) * dim, nelements_per_axis=(nel_1d,) * dim - ) - global_nelements = local_mesh.nelements - local_nelements = local_mesh.nelements + local_nelements = local_mesh.nelements discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) nodes = thaw(actx, discr.nodes()) + + if logmgr: + logmgr_add_device_name(logmgr, queue) + logmgr_add_device_memory_usage(logmgr, queue) + logmgr_add_many_discretization_quantities(logmgr, discr, dim, + extract_vars_for_logging, units_for_logging) + + logmgr.add_watches([ + ("step.max", "step = {value}, "), + ("t_sim.max", "sim time: {value:1.6e} s\n"), + ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), + ("max_pressure", "{value:1.9e})\n"), + ("t_step.max", "------- step walltime: {value:6g} s, "), + ("t_log.max", "log walltime: {value:6g} s") + ]) + uniform_state = initializer(nodes) acoustic_pulse = AcousticPulse(dim=dim, amplitude=1.0, width=.1, center=orig) - current_state = acoustic_pulse(x_vec=nodes, cv=uniform_state, eos=eos) + if rst_step: + current_t = restart_data["t"] + current_step = rst_step + current_state = restart_data["state"] + else: + # Set the current state from time 0 + current_state = acoustic_pulse(x_vec=nodes, cv=uniform_state, eos=eos) visualizer = make_visualizer(discr) @@ -145,70 +205,120 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): dt=current_dt, cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) + def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + if do_viz: + my_write_viz(cv, step, t) + if do_restart: + my_write_restart(state=cv, step=step, t=t) + if message is None: + message = "Fatal simulation errors detected." + raise RuntimeError(message) + + def my_write_viz(cv, step, t, dv=None): + viz_fields = [("cv", cv)] + if dv is not None: + viz_fields.append(("dv", dv)) + from mirgecom.simutils import write_visfile + write_visfile(discr, viz_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + def my_write_restart(state, step, t): + rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "global_nelements": global_nelements, + "num_parts": num_parts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(dv, dt): + health_error = False + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure) \ + or check_range_local(discr, "vol", dv.pressure, .8, 1.5): + health_error = True + logger.info(f"{rank=}: Invalid pressure data found.") + return health_error + def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) - def my_checkpoint(step, t, dt, state): + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + + def my_pre_step(step, t, dt, state): + dv = None + pre_step_errors = False + + if logmgr: + logmgr.tick_before() + from mirgecom.simutil import check_step - do_status = check_step(step=step, interval=nstatus) do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) - if do_status or do_viz or do_health: - pressure = eos.pressure(state) - io_fields = [ - ("cv", state), - ("pressure", pressure) - ] - - if do_status: # This is bad, logging already completely replaces this - from mirgecom.io import make_status_message - dv = eos.dependent_vars(state) - # this is also bad - no option for user customization, field selection - status_msg = make_status_message(discr=discr, t=t, step=step, dt=dt, - cfl=current_cfl, dependent_vars=dv) - if rank == 0: - logger.info(status_msg) + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False - errored = False if do_health: - from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(discr, "vol", pressure) \ - or check_range_local(discr, "vol", pressure, min_value=.8, - max_value=1.5): - errored = True - message = "Invalid pressure data found.\n" - comm = discr.mpi_communicator + dv = eos.dependent_vars(state) + local_health_error = my_health_check(dv, dt) + health_errors = False if comm is not None: - errored = comm.allreduce(errored, op=MPI.LOR) - if errored: - if rank == 0: - logger.info("Fluid solution failed health check.") - logger.info(message) # do this on all ranks + health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + if health_errors and rank == 0: + logger.info("Fluid solution failed health check.") + pre_step_errors = pre_step_errors or health_errors + + if do_restart: + my_write_restart(state, step, t) - if do_viz or errored: - from mirgecom.simutil import write_visfile - write_visfile(discr, io_fields, visualizer, vizname=casename, - step=step, t=t, overwrite=True) + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + my_write_viz(cv=state, dv=dv, step=step, t=t) - if errored: - raise RuntimeError("Error detected by user checkpoint, exiting.") + if pre_step_errors: + my_graceful_exit(cv=state, step=step, t=t, + do_viz=(not do_viz), do_restart=(not do_restart), + message="Error detected at prestep, exiting.") return state, dt current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, + pre_step_callback=my_pre_step, + post_step_callback=my_post_step, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) - # if current_t != checkpoint_t: + finish_tol = 1e-16 + if np.abs(current_t - t_final) > finish_tol: + my_graceful_exit(cv=current_state, step=current_step, t=current_t, + do_viz=True, do_restart=True, + message="Simulation timestepping did not complete.") + + # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") - my_checkpoint(current_step, t=current_t, - dt=(current_t - checkpoint_t), - state=current_state) + final_dv = eos.dependent_vars(current_state) + my_write_viz(cv=current_state, dv=final_dv, step=current_step, t=current_t) + my_write_restart(current_state, current_step, current_t) if __name__ == "__main__": From ff489e1db9a18c71f7862aaf34ec12590042df09 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 6 Jul 2021 20:06:39 -0500 Subject: [PATCH 215/385] Fix simutil import --- examples/pulse-mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 192315ccf..317751ef1 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -220,7 +220,7 @@ def my_write_viz(cv, step, t, dv=None): viz_fields = [("cv", cv)] if dv is not None: viz_fields.append(("dv", dv)) - from mirgecom.simutils import write_visfile + from mirgecom.simutil import write_visfile write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) From aa075137632c29a998c0b78da01de50ff36a9d26 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 7 Jul 2021 08:50:32 -0500 Subject: [PATCH 216/385] Fix up steppers to call get_timestep properly --- mirgecom/steppers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 9eef90a68..765d1872e 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -89,7 +89,7 @@ def _advance_state_stepper_func(rhs, timestepper, logmgr.tick_before() if get_timestep: - dt = get_timestep(state=state) + dt = get_timestep(state=state, t=t, dt=dt) if pre_step_callback is not None: state, dt = pre_step_callback(state=state, step=istep, t=t, dt=dt) @@ -164,13 +164,14 @@ def _advance_state_leap(rhs, timestepper, state, # Generate code for Leap method. if get_timestep: - dt = get_timestep(state=state) + dt = get_timestep(state=state, t=t, dt=dt) + stepper_cls = generate_singlerate_leap_advancer(timestepper, component_id, rhs, t, dt, state) while t < t_final: if get_timestep: - dt = get_timestep(state=state) + dt = get_timestep(state=state, t=t, dt=dt) if dt < 0: return istep, t, state From 1464b9b8847181e3ada72d014334ce9b091b7060 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 7 Jul 2021 08:51:22 -0500 Subject: [PATCH 217/385] Fix up inviscid_sim_timestep to properly consider t_remaining --- mirgecom/simutil.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index d88bae91e..50bac0922 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -78,12 +78,11 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, t_final, constant_cfl=False): """Return the maximum stable dt.""" mydt = dt + t_remaining = max(0, t_final - t) if constant_cfl is True: mydt = get_inviscid_timestep(discr=discr, cv=state, cfl=cfl, eos=eos) - if (t + mydt) > t_final: - mydt = t_final - t - return mydt + return min(t_remaining, mydt) def write_visfile(discr, io_fields, visualizer, vizname, From 3dfdb216ae0e644fdda5a49d62c2363f775d20b8 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 7 Jul 2021 09:50:54 -0500 Subject: [PATCH 218/385] Modernize examples. --- examples/autoignition-mpi.py | 11 +- examples/lump-mpi.py | 17 +- examples/mixture-mpi.py | 15 +- examples/pulse-mpi.py | 24 ++- examples/scalar-lump-mpi.py | 290 +++++++++++++++++++++++++--------- examples/sod-mpi.py | 292 ++++++++++++++++++++++++++--------- examples/vortex-mpi.py | 241 ++++++++++++++++++++--------- 7 files changed, 646 insertions(+), 244 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index e0ce7429d..5bd53a251 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -294,8 +294,8 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, f" {eq_pressure=}, {eq_temperature=}," f" {eq_density=}, {eq_mass_fractions=}") - get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, - dt=current_dt, cfl=current_cfl, eos=eos, + get_timestep = partial(inviscid_sim_timestep, discr=discr, + cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): @@ -410,7 +410,7 @@ def my_pre_step(step, t, dt, state): current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, - post_step_callback=my_post_step, + post_step_callback=my_post_step, dt=current_dt, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) @@ -427,6 +427,11 @@ def my_pre_step(step, t, dt, state): step=current_step, t=current_t) my_write_restart(current_state, current_step, current_t) + if logmgr: + logmgr.close() + elif use_profiling: + print(actx.tabulate_profiling_data()) + if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 079c4e47f..51ce381b9 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -118,8 +118,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step - box_ll = -5.0 - box_ur = 5.0 rst_path = "restart_data/" rst_pattern = ( @@ -135,6 +133,8 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, global_nelements = restart_data["global_nelements"] assert restart_data["nparts"] == nparts else: # generate the grid from scratch + box_ll = -5.0 + box_ur = 5.0 from meshmode.mesh.generation import generate_regular_rect_mesh generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) @@ -186,8 +186,8 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, if rank == 0: logger.info(init_message) - get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, - dt=current_dt, cfl=current_cfl, eos=eos, + get_timestep = partial(inviscid_sim_timestep, discr=discr, + cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): @@ -240,7 +240,7 @@ def my_health_check(state, dv, exact_lump): if max(component_errors) > exittol: health_error = True if rank == 0: - logger.info("Solution diverged from exact_lump.") + logger.info("Solution diverged from exact soln.") return health_error @@ -319,7 +319,7 @@ def my_pre_step(step, t, dt, state): current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, - post_step_callback=my_post_step, + post_step_callback=my_post_step, dt=current_dt, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) @@ -340,6 +340,11 @@ def my_pre_step(step, t, dt, state): resid=final_resid, step=current_step, t=current_t) my_write_restart(current_state, current_step, current_t) + if logmgr: + logmgr.close() + elif use_profiling: + print(actx.tabulate_profiling_data()) + if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 7bb3e6993..5c5076546 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -206,8 +206,8 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, if rank == 0: logger.info(init_message) - get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, - dt=current_dt, cfl=current_cfl, eos=eos, + get_timestep = partial(inviscid_sim_timestep, discr=discr, + cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): @@ -339,7 +339,7 @@ def my_pre_step(step, t, dt, state): current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, - post_step_callback=my_post_step, + post_step_callback=my_post_step, dt=current_dt, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) @@ -350,14 +350,19 @@ def my_pre_step(step, t, dt, state): message="Simulation timestepping did not complete.") # Dump the final data + if rank == 0: + logger.info("Checkpointing final state ...") final_dv = eos.dependent_vars(current_state) final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) final_resid = current_state - final_exact my_write_viz(cv=current_state, dv=final_dv, exact_mix=final_exact, resid=final_resid, step=current_step, t=current_t) my_write_restart(current_state, current_step, current_t) - if rank == 0: - logger.info("Checkpointing final state ...") + + if logmgr: + logmgr.close() + elif use_profiling: + print(actx.tabulate_profiling_data()) if __name__ == "__main__": diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 317751ef1..74c4b674e 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -71,16 +71,11 @@ @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_logmgr=False, +def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename="pulse", rst_step=None, rst_name=None): """Drive the example.""" cl_ctx = ctx_factory() - queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - - cl_ctx = ctx_factory() if casename is None: casename = "mirgecom" @@ -111,12 +106,10 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, current_cfl = 1.0 vel = np.zeros(shape=(dim,)) orig = np.zeros(shape=(dim,)) - # vel[:dim] = 1.0 current_dt = .01 current_t = 0 eos = IdealSingleGas() initializer = Lump(dim=dim, center=orig, velocity=vel, rhoamp=0.0) - casename = "pulse" boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} wall = AdiabaticSlipBoundary() boundaries = {BTAG_ALL: wall} @@ -201,9 +194,9 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, if rank == 0: logger.info(init_message) - get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, - dt=current_dt, cfl=current_cfl, eos=eos, - t_final=t_final, constant_cfl=constant_cfl) + get_timestep = partial(inviscid_sim_timestep, discr=discr, + cfl=current_cfl, eos=eos, t_final=t_final, + constant_cfl=constant_cfl) def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): if rank == 0: @@ -262,7 +255,7 @@ def my_post_step(step, t, dt, state): def my_pre_step(step, t, dt, state): dv = None pre_step_errors = False - + print(f"{step=},{t=},{dt=}") if logmgr: logmgr.tick_before() @@ -303,7 +296,7 @@ def my_pre_step(step, t, dt, state): current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, - post_step_callback=my_post_step, + post_step_callback=my_post_step, dt=current_dt, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) @@ -320,6 +313,11 @@ def my_pre_step(step, t, dt, state): my_write_viz(cv=current_state, dv=final_dv, step=current_step, t=current_t) my_write_restart(current_state, current_step, current_t) + if logmgr: + logmgr.close() + elif use_profiling: + print(actx.tabulate_profiling_data()) + if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 4cf40dd8b..87e3bdb23 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -51,32 +51,62 @@ from mirgecom.initializers import MulticomponentLump from mirgecom.eos import IdealSingleGas +from logpyle import IntervalTimer, set_dt +from mirgecom.euler import extract_vars_for_logging, units_for_logging +from mirgecom.profiling import PyOpenCLProfilingArrayContext +from mirgecom.logging_quantities import ( + initialize_logmgr, + logmgr_add_many_discretization_quantities, + logmgr_add_device_name, + logmgr_add_device_memory_usage, + set_sim_state +) logger = logging.getLogger(__name__) @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False): +def main(ctx_factory=cl.create_some_context, use_leap=False, + use_profiling=False, rst_step=None, rst_name=None, + casename="lumpy-scalars", use_logmgr=True): """Drive example.""" cl_ctx = ctx_factory() - queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, + + if casename is None: + casename = "mirgecom" + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + nparts = comm.Get_size() + + logmgr = initialize_logmgr(use_logmgr, + filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) + + if use_profiling: + queue = cl.CommandQueue(cl_ctx, + properties=cl.command_queue_properties.PROFILING_ENABLE) + actx = PyOpenCLProfilingArrayContext(queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), + logmgr=logmgr) + else: + queue = cl.CommandQueue(cl_ctx) + actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) dim = 3 nel_1d = 16 order = 3 - exittol = .09 t_final = 0.01 current_cfl = 1.0 current_dt = .001 current_t = 0 constant_cfl = False nstatus = 1 + nrestart = 5 nviz = 1 nhealth = 1 rank = 0 - checkpoint_t = current_t current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder @@ -86,10 +116,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): box_ll = -5.0 box_ur = 5.0 - from mpi4py import MPI - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - nspecies = 4 centers = make_obj_array([np.zeros(shape=(dim,)) for i in range(nspecies)]) spec_y0s = np.ones(shape=(nspecies,)) @@ -104,19 +130,59 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): spec_amplitudes=spec_amplitudes) boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} - casename = "mixture-lump" - - from meshmode.mesh.generation import generate_regular_rect_mesh - generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,) * dim, - b=(box_ur,) * dim, nelements_per_axis=(nel_1d,) * dim) - local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) - local_nelements = local_mesh.nelements + rst_path = "restart_data/" + rst_pattern = ( + rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" + ) + if rst_step: # read the grid from restart data + rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) + + from mirgecom.restart import read_restart_data + restart_data = read_restart_data(actx, rst_fname) + local_mesh = restart_data["local_mesh"] + local_nelements = local_mesh.nelements + global_nelements = restart_data["global_nelements"] + assert restart_data["nparts"] == nparts + else: # generate the grid from scratch + box_ll = -5.0 + box_ur = 5.0 + from meshmode.mesh.generation import generate_regular_rect_mesh + generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, + b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) + local_mesh, global_nelements = generate_and_distribute_mesh(comm, + generate_mesh) + local_nelements = local_mesh.nelements discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) nodes = thaw(actx, discr.nodes()) - current_state = initializer(nodes) + + if logmgr: + logmgr_add_device_name(logmgr, queue) + logmgr_add_device_memory_usage(logmgr, queue) + logmgr_add_many_discretization_quantities(logmgr, discr, dim, + extract_vars_for_logging, units_for_logging) + + logmgr.add_watches([ + ("step.max", "step = {value}, "), + ("t_sim.max", "sim time: {value:1.6e} s\n"), + ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), + ("max_pressure", "{value:1.9e})\n"), + ("t_step.max", "------- step walltime: {value:6g} s, "), + ("t_log.max", "log walltime: {value:6g} s") + ]) + + vis_timer = IntervalTimer("t_vis", "Time spent visualizing") + logmgr.add_quantity(vis_timer) + + if rst_step: + current_t = restart_data["t"] + current_step = rst_step + current_state = restart_data["state"] + else: + # Set the current state from time 0 + current_state = initializer(nodes) visualizer = make_visualizer(discr) initname = initializer.__class__.__name__ @@ -131,82 +197,164 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): if rank == 0: logger.info(init_message) - get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, - dt=current_dt, cfl=current_cfl, eos=eos, + get_timestep = partial(inviscid_sim_timestep, discr=discr, + cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) - def my_rhs(t, state): - return euler_operator(discr, cv=state, t=t, - boundaries=boundaries, eos=eos) + def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + if do_viz: + my_write_viz(cv, step, t) + if do_restart: + my_write_restart(state=cv, step=step, t=t) + if message is None: + message = "Fatal simulation errors detected." + raise RuntimeError(message) + + def my_write_viz(cv, step, t, dv=None, exact_lump=None, resid=None): + viz_fields = [("cv", cv)] + if dv is not None: + viz_fields.append(("dv", dv)) + if exact_lump is not None: + viz_fields.append(("exact", exact_lump)) + if resid is not None: + viz_fields.append(("residual", resid)) + from mirgecom.simutil import write_visfile + write_visfile(discr, viz_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + def my_write_restart(state, step, t): + rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "global_nelements": global_nelements, + "num_parts": nparts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(state, dv, exact_lump): + health_error = False + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure) \ + or check_range_local(discr, "vol", dv.pressure, .9, 1.1): + health_error = True + logger.info(f"{rank=}: Invalid pressure data found.") + + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact_lump) + exittol = .09 + if max(component_errors) > exittol: + health_error = True + if rank == 0: + logger.info("Solution diverged from exact_lump.") + + return health_error + + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + + def my_pre_step(step, t, dt, state): + dv = None + exact_lump = None + pre_step_errors = False + + if logmgr: + logmgr.tick_before() - def my_checkpoint(step, t, dt, state): from mirgecom.simutil import check_step - do_status = check_step(step=step, interval=nstatus) do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) + do_status = check_step(step=step, interval=nstatus) - if do_status or do_viz or do_health: - from mirgecom.simutil import compare_fluid_solutions + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False + + if do_health: dv = eos.dependent_vars(state) - exact_mix = initializer(x_vec=nodes, eos=eos, t=t) - component_errors = compare_fluid_solutions(discr, state, exact_mix) - resid = state - exact_mix - io_fields = [ - ("cv", state), - ("dv", dv), - ("exact_mix", exact_mix), - ("resid", resid) - ] - - if do_status: # This is bad, logging already completely replaces this - from mirgecom.io import make_status_message - status_msg = make_status_message(discr=discr, t=t, step=step, dt=dt, - cfl=current_cfl, dependent_vars=dv) - status_msg += ( - "\n------- errors=" + exact_lump = initializer(x_vec=nodes, eos=eos, t=t) + local_health_error = my_health_check(state, dv, exact_lump) + health_errors = False + if comm is not None: + health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + if health_errors and rank == 0: + logger.info("Fluid solution failed health check.") + pre_step_errors = pre_step_errors or health_errors + + if do_restart: + my_write_restart(state, step, t) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + if exact_lump is None: + exact_lump = initializer(x_vec=nodes, eos=eos, t=t) + resid = state - exact_lump + my_write_viz(cv=state, dv=dv, step=step, t=t, + exact_lump=exact_lump, resid=resid) + + if do_status: + if exact_lump is None: + exact_lump = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact_lump) + status_msg = ( + "------- errors=" + ", ".join("%.3g" % en for en in component_errors)) if rank == 0: logger.info(status_msg) - errored = False - if do_health: - from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure, min_value=.9, - max_value=1.1): - errored = True - message = "Invalid pressure data found.\n" - if max(component_errors) > exittol: - errored = True - message += "Solution errors exceed tolerance.\n" - errored = discr.mpi_communicator.allreduce(errored, op=MPI.LOR) - if errored: - if rank == 0: - logger.info("Fluid solution failed health check.") - logger.info(message) # do this on all ranks - - if do_viz or errored: - from mirgecom.simutil import write_visfile - write_visfile(discr, io_fields, visualizer, vizname=casename, - step=step, t=t, overwrite=True) - - if errored: - raise RuntimeError("Error detected by user checkpoint, exiting.") + if pre_step_errors: + my_graceful_exit(cv=state, step=step, t=t, + do_viz=(not do_viz), do_restart=(not do_restart), + message="Error detected at prestep, exiting.") return state, dt + def my_rhs(t, state): + return euler_operator(discr, cv=state, t=t, + boundaries=boundaries, eos=eos) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, + pre_step_callback=my_pre_step, dt=current_dt, + post_step_callback=my_post_step, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) - # if current_t != checkpoint_t: + finish_tol = 1e-16 + if np.abs(current_t - t_final) > finish_tol: + my_graceful_exit(cv=current_state, step=current_step, t=current_t, + do_viz=True, do_restart=True, + message="Simulation timestepping did not complete.") + + # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") - my_checkpoint(current_step, t=current_t, - dt=(current_t - checkpoint_t), - state=current_state) + + final_dv = eos.dependent_vars(current_state) + final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) + final_resid = current_state - final_exact + my_write_viz(cv=current_state, dv=final_dv, exact_lump=final_exact, + resid=final_resid, step=current_step, t=current_t) + my_write_restart(current_state, current_step, current_t) + + if logmgr: + logmgr.close() + elif use_profiling: + print(actx.tabulate_profiling_data()) if __name__ == "__main__": diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 119f8d309..286e6b07e 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -50,62 +50,123 @@ from mirgecom.initializers import SodShock1D from mirgecom.eos import IdealSingleGas +from logpyle import set_dt +from mirgecom.euler import extract_vars_for_logging, units_for_logging +from mirgecom.profiling import PyOpenCLProfilingArrayContext +from mirgecom.logging_quantities import ( + initialize_logmgr, + logmgr_add_many_discretization_quantities, + logmgr_add_device_name, + logmgr_add_device_memory_usage, + set_sim_state +) logger = logging.getLogger(__name__) @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False): +def main(ctx_factory=cl.create_some_context, use_logmgr=True, + use_leap=False, use_profiling=False, casename="sod1d", + rst_step=None, rst_name=None): """Drive the example.""" cl_ctx = ctx_factory() - queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + if casename is None: + casename = "mirgecom" + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + num_parts = comm.Get_size() + + logmgr = initialize_logmgr(use_logmgr, + filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) + + if use_profiling: + queue = cl.CommandQueue(cl_ctx, + properties=cl.command_queue_properties.PROFILING_ENABLE) + actx = PyOpenCLProfilingArrayContext(queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), + logmgr=logmgr) + else: + queue = cl.CommandQueue(cl_ctx) + actx = PyOpenCLArrayContext(queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) dim = 1 nel_1d = 24 order = 1 # tolerate large errors; case is unstable - exittol = .2 t_final = 0.01 current_cfl = 1.0 current_dt = .0001 current_t = 0 eos = IdealSingleGas() initializer = SodShock1D(dim=dim) - casename = "sod1d" boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} constant_cfl = False nstatus = 10 + nrestart = 5 nviz = 10 nhealth = 10 rank = 0 - checkpoint_t = current_t current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step - box_ll = -5.0 - box_ur = 5.0 - - from mpi4py import MPI - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - from meshmode.mesh.generation import generate_regular_rect_mesh - generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,) * dim, - b=(box_ur,) * dim, nelements_per_axis=(nel_1d,) * dim) - local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) - - local_nelements = local_mesh.nelements + rst_path = "restart_data/" + rst_pattern = ( + rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" + ) + if rst_step: # read the grid from restart data + rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) + + from mirgecom.restart import read_restart_data + restart_data = read_restart_data(actx, rst_fname) + local_mesh = restart_data["local_mesh"] + local_nelements = local_mesh.nelements + global_nelements = restart_data["global_nelements"] + assert restart_data["nparts"] == num_parts + else: # generate the grid from scratch + from meshmode.mesh.generation import generate_regular_rect_mesh + box_ll = -5.0 + box_ur = 5.0 + generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, + b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) + local_mesh, global_nelements = generate_and_distribute_mesh(comm, + generate_mesh) + local_nelements = local_mesh.nelements discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) nodes = thaw(actx, discr.nodes()) - current_state = initializer(nodes) + + if logmgr: + logmgr_add_device_name(logmgr, queue) + logmgr_add_device_memory_usage(logmgr, queue) + logmgr_add_many_discretization_quantities(logmgr, discr, dim, + extract_vars_for_logging, units_for_logging) + + logmgr.add_watches([ + ("step.max", "step = {value}, "), + ("t_sim.max", "sim time: {value:1.6e} s\n"), + ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), + ("max_pressure", "{value:1.9e})\n"), + ("t_step.max", "------- step walltime: {value:6g} s, "), + ("t_log.max", "log walltime: {value:6g} s") + ]) + + if rst_step: + current_t = restart_data["t"] + current_step = rst_step + current_state = restart_data["state"] + else: + # Set the current state from time 0 + current_state = initializer(nodes) visualizer = make_visualizer(discr) @@ -121,81 +182,164 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): if rank == 0: logger.info(init_message) - get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, - dt=current_dt, cfl=current_cfl, eos=eos, - t_final=t_final, constant_cfl=constant_cfl) + def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + if do_viz: + my_write_viz(cv, step, t) + if do_restart: + my_write_restart(state=cv, step=step, t=t) + if message is None: + message = "Fatal simulation errors detected." + raise RuntimeError(message) + + def my_write_viz(cv, step, t, dv=None, exact=None, resid=None): + viz_fields = [("cv", cv)] + if dv is not None: + viz_fields.append(("dv", dv)) + if exact is not None: + viz_fields.append(("exact_soln", exact)) + if resid is not None: + viz_fields.append(("residual", resid)) + from mirgecom.simutil import write_visfile + write_visfile(discr, viz_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + def my_write_restart(state, step, t): + rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "global_nelements": global_nelements, + "num_parts": num_parts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(state, dv, exact): + health_error = False + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure) \ + or check_range_local(discr, "vol", dv.pressure, .09, 1.1): + health_error = True + logger.info(f"{rank=}: Invalid pressure data found.") + + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + exittol = .09 + if max(component_errors) > exittol: + health_error = True + if rank == 0: + logger.info("Solution diverged from exact soln.") - def my_rhs(t, state): - return euler_operator(discr, cv=state, t=t, - boundaries=boundaries, eos=eos) + return health_error + + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + + def my_pre_step(step, t, dt, state): + dv = None + exact = None + pre_step_errors = False + + if logmgr: + logmgr.tick_before() - def my_checkpoint(step, t, dt, state): from mirgecom.simutil import check_step - do_status = check_step(step=step, interval=nstatus) do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) + do_status = check_step(step=step, interval=nstatus) - if do_status or do_viz or do_health: - from mirgecom.simutil import compare_fluid_solutions + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False + + if do_health: dv = eos.dependent_vars(state) - sod_exact = initializer(x_vec=nodes, eos=eos, t=t) - component_errors = compare_fluid_solutions(discr, state, sod_exact) - resid = state - sod_exact - io_fields = [ - ("cv", state), - ("dv", dv), - ("sod_exact", sod_exact), - ("resid", resid) - ] - - if do_status: # This is bad, logging already completely replaces this - from mirgecom.io import make_status_message - status_msg = make_status_message(discr=discr, t=t, step=step, dt=dt, - cfl=current_cfl, dependent_vars=dv) - status_msg += ( - "\n------- errors=" + exact = initializer(x_vec=nodes, eos=eos, t=t) + local_health_error = my_health_check(state, dv, exact) + health_errors = False + if comm is not None: + health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + if health_errors and rank == 0: + logger.info("Fluid solution failed health check.") + pre_step_errors = pre_step_errors or health_errors + + if do_restart: + my_write_restart(state, step, t) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + resid = state - exact + my_write_viz(cv=state, dv=dv, step=step, t=t, + exact=exact, resid=resid) + + if do_status: + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + status_msg = ( + "------- errors=" + ", ".join("%.3g" % en for en in component_errors)) if rank == 0: logger.info(status_msg) - errored = False - if do_health: - from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure, .09, 1.1): - errored = True - message = "Invalid pressure data found.\n" - if np.max(component_errors) > exittol: - errored = True - message += "Solution errors exceed tolerance.\n" - errored = discr.mpi_communicator.allreduce(errored, op=MPI.LOR) - if errored: - if rank == 0: - logger.info("Fluid solution failed health check.") - logger.info(message) # do this on all ranks - - if do_viz or errored: - from mirgecom.simutil import write_visfile - write_visfile(discr, io_fields, visualizer, vizname=casename, - step=step, t=t, overwrite=True) - - if errored: - raise RuntimeError("Error detected by user checkpoint, exiting.") + if pre_step_errors: + my_graceful_exit(cv=state, step=step, t=t, + do_viz=(not do_viz), do_restart=(not do_restart), + message="Error detected at prestep, exiting.") return state, dt + get_timestep = partial(inviscid_sim_timestep, discr=discr, + cfl=current_cfl, eos=eos, t_final=t_final, + constant_cfl=constant_cfl) + + def my_rhs(t, state): + return euler_operator(discr, cv=state, t=t, + boundaries=boundaries, eos=eos) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, + pre_step_callback=my_pre_step, dt=current_dt, + post_step_callback=my_post_step, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) - # if current_t != checkpoint_t: + finish_tol = 1e-16 + if np.abs(current_t - t_final) > finish_tol: + my_graceful_exit(cv=current_state, step=current_step, t=current_t, + do_viz=True, do_restart=True, + message="Simulation timestepping did not complete.") + + # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") - my_checkpoint(current_step, t=current_t, - dt=(current_t - checkpoint_t), - state=current_state) + + final_dv = eos.dependent_vars(current_state) + final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) + final_resid = current_state - final_exact + my_write_viz(cv=current_state, dv=final_dv, exact=final_exact, + resid=final_resid, step=current_step, t=current_t) + my_write_restart(current_state, current_step, current_t) + + if logmgr: + logmgr.close() + elif use_profiling: + print(actx.tabulate_profiling_data()) if __name__ == "__main__": diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 3a70d7716..c079101dd 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -67,16 +67,23 @@ @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=False, - use_leap=False): +def main(ctx_factory=cl.create_some_context, use_logmgr=True, + use_leap=False, use_profiling=False, casename="sod1d", + rst_step=None, rst_name=None): """Drive the example.""" + cl_ctx = ctx_factory() + + if casename is None: + casename = "mirgecom" + from mpi4py import MPI comm = MPI.COMM_WORLD + rank = comm.Get_rank() + num_parts = comm.Get_size() logmgr = initialize_logmgr(use_logmgr, - filename="vortex.sqlite", mode="wu", mpi_comm=comm) + filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) - cl_ctx = ctx_factory() if use_profiling: queue = cl.CommandQueue(cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) @@ -91,7 +98,6 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal dim = 2 nel_1d = 16 order = 3 - exittol = .1 t_final = 0.1 current_cfl = 1.0 vel = np.zeros(shape=(dim,)) @@ -104,35 +110,47 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal casename = "vortex" boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} constant_cfl = False - nstatus = 10 + nrestart = 10 + nstatus = 1 nviz = 10 nhealth = 10 - checkpoint_t = current_t current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder timestepper = RK4MethodBuilder("state") else: timestepper = rk4_step - box_ll = -5.0 - box_ur = 5.0 - - rank = comm.Get_rank() if dim != 2: raise ValueError("This example must be run with dim = 2.") - from meshmode.mesh.generation import generate_regular_rect_mesh - generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,) * dim, - b=(box_ur,) * dim, nelements_per_axis=(nel_1d,) * dim) - local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) - local_nelements = local_mesh.nelements + rst_path = "restart_data/" + rst_pattern = ( + rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" + ) + if rst_step: # read the grid from restart data + rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) + + from mirgecom.restart import read_restart_data + restart_data = read_restart_data(actx, rst_fname) + local_mesh = restart_data["local_mesh"] + local_nelements = local_mesh.nelements + global_nelements = restart_data["global_nelements"] + assert restart_data["nparts"] == num_parts + else: # generate the grid from scratch + from meshmode.mesh.generation import generate_regular_rect_mesh + box_ll = -5.0 + box_ur = 5.0 + generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, + b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) + local_mesh, global_nelements = generate_and_distribute_mesh(comm, + generate_mesh) + local_nelements = local_mesh.nelements discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) nodes = thaw(actx, discr.nodes()) - current_state = initializer(nodes) vis_timer = None @@ -162,6 +180,14 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) + if rst_step: + current_t = restart_data["t"] + current_step = rst_step + current_state = restart_data["state"] + else: + # Set the current state from time 0 + current_state = initializer(nodes) + visualizer = make_visualizer(discr) initname = initializer.__class__.__name__ @@ -176,87 +202,158 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal if rank == 0: logger.info(init_message) - get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, - dt=current_dt, cfl=current_cfl, eos=eos, - t_final=t_final, constant_cfl=constant_cfl) + def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + if do_viz: + my_write_viz(cv, step, t) + if do_restart: + my_write_restart(state=cv, step=step, t=t) + if message is None: + message = "Fatal simulation errors detected." + raise RuntimeError(message) + + def my_write_viz(cv, step, t, dv=None, exact=None, resid=None): + viz_fields = [("cv", cv)] + if dv is not None: + viz_fields.append(("dv", dv)) + if exact is not None: + viz_fields.append(("exact_soln", exact)) + if resid is not None: + viz_fields.append(("residual", resid)) + from mirgecom.simutil import write_visfile + write_visfile(discr, viz_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + def my_write_restart(state, step, t): + rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "global_nelements": global_nelements, + "num_parts": num_parts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(state, dv, exact): + health_error = False + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure) \ + or check_range_local(discr, "vol", dv.pressure, .2, 1.02): + health_error = True + logger.info(f"{rank=}: Invalid pressure data found.") + + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + exittol = .1 + if max(component_errors) > exittol: + health_error = True + if rank == 0: + logger.info("Solution diverged from exact soln.") - def my_rhs(t, state): - return euler_operator(discr, cv=state, t=t, - boundaries=boundaries, eos=eos) + return health_error - def post_step_stuff(step, t, dt, state): + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) logmgr.tick_after() return state, dt - def my_checkpoint(step, t, dt, state): - do_status = check_step(step=step, interval=nstatus) - do_viz = check_step(step=step, interval=nviz) - do_health = check_step(step=step, interval=nhealth) + def my_pre_step(step, t, dt, state): + dv = None + exact = None + pre_step_errors = False if logmgr: logmgr.tick_before() - if do_status or do_viz or do_health: - from mirgecom.simutil import compare_fluid_solutions - pressure = eos.pressure(state) - vortex_exact = initializer(x_vec=nodes, eos=eos, t=t) - component_errors = compare_fluid_solutions(discr, state, vortex_exact) + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) + do_status = check_step(step=step, interval=nstatus) + + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False + + if do_health: + dv = eos.dependent_vars(state) + exact = initializer(x_vec=nodes, eos=eos, t=t) + local_health_error = my_health_check(state, dv, exact) + health_errors = False + if comm is not None: + health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + if health_errors and rank == 0: + logger.info("Fluid solution failed health check.") + pre_step_errors = pre_step_errors or health_errors + + if do_restart: + my_write_restart(state, step, t) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + resid = state - exact + my_write_viz(cv=state, dv=dv, step=step, t=t, + exact=exact, resid=resid) if do_status: + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + status_msg = ( + "------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) if rank == 0: - logger.info( - "------- errors=" + ",".join("%.3g" % en for en - in component_errors) - ) + logger.info(status_msg) - errored = False - if do_health: - from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(discr, "vol", pressure) \ - or check_range_local(discr, "vol", pressure, .2, 1.02): - errored = True - message = "Invalid pressure data found.\n" - if np.max(component_errors) > exittol: - errored = True - message += "Solution errors exceed tolerance.\n" - errored = discr.mpi_communicator.allreduce(errored, op=MPI.LOR) - if errored: - if rank == 0: - logger.info("Fluid solution failed health check.") - logger.info(message) # do this on all ranks - - if do_viz or errored: - resid = state - vortex_exact - io_fields = [ - ("cv", state), - ("pressure", pressure), - ("vortex_exact", vortex_exact), - ("resid", resid) - ] - from mirgecom.simutil import write_visfile - write_visfile(discr, io_fields, visualizer, vizname=casename, - step=step, t=t, overwrite=True) - - if errored: - raise RuntimeError("Error detected by user checkpoint, exiting.") + if pre_step_errors: + my_graceful_exit(cv=state, step=step, t=t, + do_viz=(not do_viz), do_restart=(not do_restart), + message="Error detected at prestep, exiting.") return state, dt + get_timestep = partial(inviscid_sim_timestep, discr=discr, + cfl=current_cfl, eos=eos, t_final=t_final, + constant_cfl=constant_cfl) + + def my_rhs(t, state): + return euler_operator(discr, cv=state, t=t, + boundaries=boundaries, eos=eos) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_checkpoint, - post_step_callback=post_step_stuff, + pre_step_callback=my_pre_step, + post_step_callback=my_post_step, dt=current_dt, get_timestep=get_timestep, state=current_state, t=current_t, t_final=t_final, eos=eos, dim=dim) + finish_tol = 1e-16 + if np.abs(current_t - t_final) > finish_tol: + my_graceful_exit(cv=current_state, step=current_step, t=current_t, + do_viz=True, do_restart=True, + message="Simulation timestepping did not complete.") + + # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") - my_checkpoint(current_step, t=current_t, - dt=(current_t - checkpoint_t), - state=current_state) + + final_dv = eos.dependent_vars(current_state) + final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) + final_resid = current_state - final_exact + my_write_viz(cv=current_state, dv=final_dv, exact=final_exact, + resid=final_resid, step=current_step, t=current_t) + my_write_restart(current_state, current_step, current_t) if logmgr: logmgr.close() From 926e76a7bc42db2437cff131d023eebeae0cc610 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 7 Jul 2021 10:06:25 -0500 Subject: [PATCH 219/385] Correct misnamed restart parameter. Co-authored-by: Matt Smith --- examples/autoignition-mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 5bd53a251..21543a1e1 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -147,7 +147,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] - assert restart_data["nparts"] == nproc + assert restart_data["num_parts"] == nproc else: # generate the grid from scratch from meshmode.mesh.generation import generate_regular_rect_mesh box_ll = -0.005 From 86fadcaa176272e0a4e677e4b3782c1b1c22e236 Mon Sep 17 00:00:00 2001 From: jlevine18 Date: Wed, 7 Jul 2021 14:27:35 -0500 Subject: [PATCH 220/385] Add constant CFL mode to examples (#411) --- examples/poiseuille-mpi.py | 40 +++++++++++++++++++++++++----------- examples/pulse-mpi.py | 42 ++++++++++++++++++++++++++------------ examples/wave-eager-mpi.py | 18 ++++++++-------- examples/wave-eager.py | 23 +++++++++++---------- 4 files changed, 78 insertions(+), 45 deletions(-) diff --git a/examples/poiseuille-mpi.py b/examples/poiseuille-mpi.py index 4d9a3c4b8..da4939622 100644 --- a/examples/poiseuille-mpi.py +++ b/examples/poiseuille-mpi.py @@ -41,11 +41,13 @@ from mirgecom.fluid import make_conserved from mirgecom.navierstokes import ns_operator from mirgecom.simutil import ( - inviscid_sim_timestep, sim_checkpoint, generate_and_distribute_mesh, ExactSolutionMismatch, ) + +from mirgecom.viscous import get_viscous_timestep + from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point from mirgecom.integrators import rk4_step @@ -87,12 +89,12 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal dim = 2 order = 1 exittol = 1.0 - t_final = 1e-7 - current_cfl = 1.0 + t_final = 1e-6 + current_cfl = 0.1 current_dt = 1e-8 current_t = 0 casename = "poiseuille" - constant_cfl = False + constant_cfl = True nstatus = 1 nviz = 1 rank = 0 @@ -164,16 +166,31 @@ def poiseuille_soln(nodes, eos, cv=None, **kwargs): if rank == 0: logger.info(init_message) - get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, - dt=current_dt, cfl=current_cfl, eos=eos, - t_final=t_final, constant_cfl=constant_cfl) - def my_rhs(t, state): return ns_operator(discr, eos=eos, boundaries=boundaries, cv=state, t=t) def my_checkpoint(step, t, dt, state): + t_remaining = max(0, t_final - t) + checkpoint_cfl = current_cfl + viz_fields = [] + + if constant_cfl is True: + dt = ( + current_cfl * get_viscous_timestep(discr=discr, eos=eos, cv=state) + ) + viz_fields.append(("local_dt", dt)) + from grudge.op import nodal_min + dt = nodal_min(discr, "vol", dt) + else: + from mirgecom.viscous import get_viscous_cfl + cfl_field = get_viscous_cfl(discr, eos, dt, state) + viz_fields.append(("cfl", cfl_field)) + from grudge.op import nodal_max + checkpoint_cfl = nodal_max(discr, "vol", cfl_field) + + dt = min(dt, t_remaining) return sim_checkpoint(discr, visualizer, eos, cv=state, - vizname=casename, step=step, + vizname=casename, step=step, cfl=checkpoint_cfl, t=t, dt=dt, nstatus=nstatus, nviz=nviz, exittol=exittol, constant_cfl=constant_cfl, comm=comm, vis_timer=vis_timer, overwrite=True) @@ -181,9 +198,8 @@ def my_checkpoint(step, t, dt, state): try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=my_checkpoint, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final, eos=eos, + checkpoint=my_checkpoint, state=current_state, + dt=current_dt, t=current_t, t_final=t_final, eos=eos, dim=dim) except ExactSolutionMismatch as ex: current_step = ex.step diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 658364d58..43cdb8e2a 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -39,12 +39,13 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( - inviscid_sim_timestep, generate_and_distribute_mesh, sim_checkpoint, ExactSolutionMismatch, ) +from mirgecom.viscous import get_viscous_timestep + from mirgecom.io import make_init_message from mirgecom.integrators import rk4_step @@ -72,8 +73,8 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): order = 1 exittol = 2e-2 exittol = 100.0 - t_final = 0.1 - current_cfl = 1.0 + t_final = .5 + current_cfl = 0.38 vel = np.zeros(shape=(dim,)) orig = np.zeros(shape=(dim,)) # vel[:dim] = 1.0 @@ -84,7 +85,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): casename = "pulse" wall = AdiabaticSlipBoundary() boundaries = {BTAG_ALL: wall} - constant_cfl = False + constant_cfl = True nstatus = 10 nviz = 10 rank = 0 @@ -141,26 +142,41 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): if rank == 0: logger.info(init_message) - get_timestep = partial(inviscid_sim_timestep, discr=discr, t=current_t, - dt=current_dt, cfl=current_cfl, eos=eos, - t_final=t_final, constant_cfl=constant_cfl) - def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) def my_checkpoint(step, t, dt, state): + t_remaining = max(0, t_final - t) + checkpoint_cfl = current_cfl + viz_fields = [] + + if constant_cfl is True: + dt = ( + current_cfl * get_viscous_timestep(discr=discr, eos=eos, cv=state) + ) + viz_fields.append(("local_dt", dt)) + from grudge.op import nodal_min + dt = nodal_min(discr, "vol", dt) + else: + from mirgecom.viscous import get_viscous_cfl + cfl_field = get_viscous_cfl(discr, eos, dt, state) + viz_fields.append(("cfl", cfl_field)) + from grudge.op import nodal_max + checkpoint_cfl = nodal_max(discr, "vol", cfl_field) + + dt = min(dt, t_remaining) return sim_checkpoint(discr, visualizer, eos, cv=state, - vizname=casename, step=step, + vizname=casename, step=step, cfl=checkpoint_cfl, t=t, dt=dt, nstatus=nstatus, nviz=nviz, - exittol=exittol, constant_cfl=constant_cfl, comm=comm) + exittol=exittol, constant_cfl=constant_cfl, + comm=comm, viz_fields=viz_fields) try: (current_step, current_t, current_state) = \ advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=my_checkpoint, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final) + checkpoint=my_checkpoint, state=current_state, + dt=current_dt, t=current_t, t_final=t_final) except ExactSolutionMismatch as ex: current_step = ex.step current_t = ex.t diff --git a/examples/wave-eager-mpi.py b/examples/wave-eager-mpi.py index b88eb232e..c60896f42 100644 --- a/examples/wave-eager-mpi.py +++ b/examples/wave-eager-mpi.py @@ -69,6 +69,9 @@ def main(snapshot_pattern="wave-eager-{step:04d}-{rank:04d}.pkl", restart_step=N actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + current_cfl = .485 + wave_speed = 1.0 + from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.Get_rank() @@ -113,14 +116,11 @@ def main(snapshot_pattern="wave-eager-{step:04d}-{rank:04d}.pkl", restart_step=N discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) - if local_mesh.dim == 2: - # no deep meaning here, just a fudge factor - dt = 0.7 / (nel_1d*order**2) - elif dim == 3: - # no deep meaning here, just a fudge factor - dt = 0.4 / (nel_1d*order**2) - else: - raise ValueError("don't have a stable time step guesstimate") + from grudge.dt_utils import characteristic_lengthscales + dt = current_cfl * characteristic_lengthscales(actx, discr) / wave_speed + + from grudge.op import nodal_min + dt = nodal_min(discr, "vol", dt) t_final = 3 @@ -152,7 +152,7 @@ def main(snapshot_pattern="wave-eager-{step:04d}-{rank:04d}.pkl", restart_step=N vis = make_visualizer(discr) def rhs(t, w): - return wave_operator(discr, c=1, w=w) + return wave_operator(discr, c=wave_speed, w=w) while t < t_final: # restart must happen at beginning of step diff --git a/examples/wave-eager.py b/examples/wave-eager.py index 7f1ce66c6..c11e6a1c5 100644 --- a/examples/wave-eager.py +++ b/examples/wave-eager.py @@ -72,6 +72,10 @@ def main(use_profiling=False): dim = 2 nel_1d = 16 + + current_cfl = .485 + wave_speed = 1.0 + from meshmode.mesh.generation import generate_regular_rect_mesh mesh = generate_regular_rect_mesh( @@ -81,18 +85,15 @@ def main(use_profiling=False): order = 3 - if dim == 2: - # no deep meaning here, just a fudge factor - dt = 0.7 / (nel_1d * order ** 2) - elif dim == 3: - # no deep meaning here, just a fudge factor - dt = 0.4 / (nel_1d * order ** 2) - else: - raise ValueError("don't have a stable time step guesstimate") + discr = EagerDGDiscretization(actx, mesh, order=order) - print("%d elements" % mesh.nelements) + from grudge.dt_utils import characteristic_lengthscales + dt = current_cfl * characteristic_lengthscales(actx, discr) / wave_speed - discr = EagerDGDiscretization(actx, mesh, order=order) + from grudge.op import nodal_min + dt = nodal_min(discr, "vol", dt) + + print("%d elements" % mesh.nelements) fields = flat_obj_array( bump(actx, discr), @@ -102,7 +103,7 @@ def main(use_profiling=False): vis = make_visualizer(discr) def rhs(t, w): - return wave_operator(discr, c=1, w=w) + return wave_operator(discr, c=wave_speed, w=w) t = 0 t_final = 3 From 992e79555baf6ba108ae45f3bb34267befd00f9e Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 7 Jul 2021 14:35:11 -0500 Subject: [PATCH 221/385] Ignore exception style issue. --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 056de71f4..6a5a17dfd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [flake8] -ignore = E127,E128,E123,E226,E241,W503 +ignore = E127,E128,E123,E226,E241,W503,N818 exclude = .*,\#* max-line-length=85 From f0e65f3fdff4368e30fb0f03b1a7dd212bb0c4ce Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 7 Jul 2021 19:54:49 -0500 Subject: [PATCH 222/385] Turn off profiling in vortex to speed up CI. --- examples/vortex-mpi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 76650ea3d..e080c38fc 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -91,7 +91,7 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal nel_1d = 16 order = 3 exittol = .1 - t_final = 0.1 + t_final = 0.01 current_cfl = 1.0 vel = np.zeros(shape=(dim,)) orig = np.zeros(shape=(dim,)) @@ -223,7 +223,7 @@ def my_checkpoint(step, t, dt, state): if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) - use_profiling = True + use_profiling = False use_logging = True use_leap = False From 6445e28e89777f5f451c7d93e5a4ee493b5436eb Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 7 Jul 2021 19:57:05 -0500 Subject: [PATCH 223/385] Reduce the number of steps in scalar lump to speed up CI. --- examples/scalar-lump-mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 9de87b9f6..151bb4dd0 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -69,7 +69,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): nel_1d = 16 order = 3 exittol = .09 - t_final = 0.01 + t_final = 0.005 current_cfl = 1.0 current_dt = .001 current_t = 0 From 82eb938750f918bfd3b0199a4dcb77a6e37e29dc Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 7 Jul 2021 19:58:17 -0500 Subject: [PATCH 224/385] Reduce number of steps to speed up CI. --- examples/vortex-mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index c079101dd..bb170b7d5 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -98,7 +98,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, dim = 2 nel_1d = 16 order = 3 - t_final = 0.1 + t_final = 0.01 current_cfl = 1.0 vel = np.zeros(shape=(dim,)) orig = np.zeros(shape=(dim,)) From 47d41d7a3470ceb2a8844399cacad01df974302b Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 8 Jul 2021 04:33:02 -0500 Subject: [PATCH 225/385] Turn off profiling, reduce number of steps to improve CI time --- examples/vortex-mpi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index ea36a3aa7..6cb8951d1 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -91,7 +91,7 @@ def main(ctx_factory=cl.create_some_context, use_profiling=False, use_logmgr=Fal nel_1d = 16 order = 3 exittol = .1 - t_final = 0.1 + t_final = 0.01 current_cfl = 1.0 vel = np.zeros(shape=(dim,)) orig = np.zeros(shape=(dim,)) @@ -220,7 +220,7 @@ def my_checkpoint(step, t, dt, state): if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) - use_profiling = True + use_profiling = False use_logging = True use_leap = False From ef1ca551eb7ec3fd28e8fdc103e6be8793d0b9ce Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 8 Jul 2021 04:33:46 -0500 Subject: [PATCH 226/385] Reduce number of steps to improve CI time. --- examples/scalar-lump-mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index d2dc0b112..39965272f 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -69,7 +69,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False): nel_1d = 16 order = 3 exittol = .09 - t_final = 0.01 + t_final = 0.005 current_cfl = 1.0 current_dt = .001 current_t = 0 From 02b37d892773bc833e465678e9252c13ef7af97d Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 8 Jul 2021 10:09:40 -0500 Subject: [PATCH 227/385] Unify call signatures of user-defined utilities. --- examples/autoignition-mpi.py | 38 ++++++++++--------- examples/lump-mpi.py | 63 +++++++++++++++++--------------- examples/mixture-mpi.py | 68 ++++++++++++++++++---------------- examples/pulse-mpi.py | 37 ++++++++++--------- examples/scalar-lump-mpi.py | 71 +++++++++++++++++++----------------- examples/sod-mpi.py | 47 +++++++++++++----------- examples/vortex-mpi.py | 49 +++++++++++++------------ 7 files changed, 196 insertions(+), 177 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 21543a1e1..ff4c56a2e 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -298,27 +298,30 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) - def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, + message=None): if rank == 0: logger.info("Errors detected; attempting graceful exit.") if do_viz: - my_write_viz(cv, step, t) + my_write_viz(step=step, t=t, state=state) if do_restart: - my_write_restart(state=cv, step=step, t=t) + my_write_restart(step=step, t=t, state=state) if message is None: message = "Fatal simulation errors detected." raise RuntimeError(message) - def my_write_viz(cv, step, t, dv=None, production_rates=None): - viz_fields = [("cv", cv)] - if dv is not None: - viz_fields.append(("dv", dv)) - if production_rates is not None: - viz_fields.append(("production_rates", production_rates)) + def my_write_viz(step, t, state, dv=None, production_rates=None): + if dv is None: + dv = eos.dependent_vars(state) + if production_rates is None: + production_rates = eos.get_production_rates(state) + viz_fields = [("cv", state), + ("dv", dv), + ("production_rates", production_rates)] write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) - def my_write_restart(state, step, t): + def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) rst_data = { "local_mesh": local_mesh, @@ -382,22 +385,21 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - local_health_error = my_health_check(dv, dt) - health_errors = False + health_errors = my_health_check(dv, dt) if comm is not None: - health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + health_errors = comm.allreduce(health_errors, op=MPI.LOR) if health_errors and rank == 0: logger.info("Fluid solution failed health check.") pre_step_errors = pre_step_errors or health_errors if do_restart: - my_write_restart(state, step, t) + my_write_restart(step=step, t=t, state=state) if do_viz: production_rates = eos.get_production_rates(state) if dv is None: dv = eos.dependent_vars(state) - my_write_viz(cv=state, dv=dv, step=step, t=t, + my_write_viz(step=step, t=t, state=state, dv=dv, production_rates=production_rates) if pre_step_errors: @@ -423,9 +425,9 @@ def my_pre_step(step, t, dt, state): # Dump the final data final_dv = eos.dependent_vars(current_state) final_dm = eos.get_production_rates(current_state) - my_write_viz(cv=current_state, dv=final_dv, production_rates=final_dm, - step=current_step, t=current_t) - my_write_restart(current_state, current_step, current_t) + my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, + production_rates=final_dm) + my_write_restart(step=current_step, t=current_t, state=current_state) if logmgr: logmgr.close() diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 51ce381b9..de6f723b1 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -190,25 +190,29 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) - def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, + message=None): if rank == 0: logger.info("Errors detected; attempting graceful exit.") if do_viz: - my_write_viz(cv, step, t) + my_write_viz(step=step, t=t, state=state) if do_restart: - my_write_restart(state=cv, step=step, t=t) + my_write_restart(step=step, t=t, state=state) if message is None: message = "Fatal simulation errors detected." raise RuntimeError(message) - def my_write_viz(cv, step, t, dv=None, exact_lump=None, resid=None): - viz_fields = [("cv", cv)] - if dv is not None: - viz_fields.append(("dv", dv)) - if exact_lump is not None: - viz_fields.append(("exact", exact_lump)) - if resid is not None: - viz_fields.append(("residual", resid)) + def my_write_viz(step, t, state, dv=None, exact=None, resid=None): + if dv is None: + dv = eos.dependent_vars(state) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + if resid is None: + resid = state - exact + viz_fields = [("cv", state), + ("dv", dv), + ("exact", exact), + ("residual", resid)] from mirgecom.simutil import write_visfile write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) @@ -226,7 +230,7 @@ def my_write_restart(state, step, t): from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) - def my_health_check(state, dv, exact_lump): + def my_health_check(dv, state, exact): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ @@ -235,7 +239,7 @@ def my_health_check(state, dv, exact_lump): logger.info(f"{rank=}: Invalid pressure data found.") from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact_lump) + component_errors = compare_fluid_solutions(discr, state, exact) exittol = .09 if max(component_errors) > exittol: health_error = True @@ -259,7 +263,7 @@ def my_post_step(step, t, dt, state): def my_pre_step(step, t, dt, state): dv = None - exact_lump = None + exact = None pre_step_errors = False if logmgr: @@ -277,32 +281,31 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - exact_lump = initializer(x_vec=nodes, eos=eos, t=t) - local_health_error = my_health_check(state, dv, exact_lump) - health_errors = False + exact = initializer(x_vec=nodes, eos=eos, t=t) + health_errors = my_health_check(dv=dv, state=state, exact=exact) if comm is not None: - health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + health_errors = comm.allreduce(health_errors, op=MPI.LOR) if health_errors and rank == 0: logger.info("Fluid solution failed health check.") pre_step_errors = pre_step_errors or health_errors if do_restart: - my_write_restart(state, step, t) + my_write_restart(step=step, t=t, state=state) if do_viz: if dv is None: dv = eos.dependent_vars(state) - if exact_lump is None: - exact_lump = initializer(x_vec=nodes, eos=eos, t=t) - resid = state - exact_lump - my_write_viz(cv=state, dv=dv, step=step, t=t, - exact_lump=exact_lump, resid=resid) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + resid = state - exact + my_write_viz(step=step, t=t, dv=dv, state=state, exact=exact, + resid=resid) if do_status: - if exact_lump is None: - exact_lump = initializer(x_vec=nodes, eos=eos, t=t) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact_lump) + component_errors = compare_fluid_solutions(discr, state, exact) status_msg = ( "------- errors=" + ", ".join("%.3g" % en for en in component_errors)) @@ -336,9 +339,9 @@ def my_pre_step(step, t, dt, state): final_dv = eos.dependent_vars(current_state) final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) final_resid = current_state - final_exact - my_write_viz(cv=current_state, dv=final_dv, exact_lump=final_exact, - resid=final_resid, step=current_step, t=current_t) - my_write_restart(current_state, current_step, current_t) + my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, + exact=final_exact, resid=final_resid) + my_write_restart(step=current_step, t=current_t, state=current_state) if logmgr: logmgr.close() diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 5c5076546..f61e740be 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -210,30 +210,35 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) - def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, + message=None): if rank == 0: logger.info("Errors detected; attempting graceful exit.") if do_viz: - my_write_viz(cv, step, t) + my_write_viz(step=step, t=t, state=state) if do_restart: - my_write_restart(state=cv, step=step, t=t) + my_write_restart(step=step, t=t, state=state) if message is None: message = "Fatal simulation errors detected." raise RuntimeError(message) - def my_write_viz(cv, step, t, dv=None, exact_mix=None, resid=None): - viz_fields = [("cv", cv)] - if dv is not None: - viz_fields.append(("dv", dv)) - if exact_mix is not None: - viz_fields.append(("exact", exact_mix)) - if resid is not None: - viz_fields.append(("residual", resid)) + def my_write_viz(step, t, state, dv=None, exact=None, resid=None): + viz_fields = [("cv", state)] + if dv is None: + dv = eos.dependent_vars(state) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + if resid is None: + resid = state - exact + viz_fields = [("cv", state), + ("dv", dv), + ("exact_soln", exact), + ("residual", resid)] from mirgecom.simutil import write_visfile write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) - def my_write_restart(state, step, t): + def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) rst_data = { "local_mesh": local_mesh, @@ -246,7 +251,7 @@ def my_write_restart(state, step, t): from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) - def my_health_check(state, dv, exact_mix): + def my_health_check(state, dv, exact): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ @@ -255,12 +260,12 @@ def my_health_check(state, dv, exact_mix): logger.info(f"{rank=}: Invalid pressure data found.") from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact_mix) + component_errors = compare_fluid_solutions(discr, state, exact) exittol = .09 if max(component_errors) > exittol: health_error = True if rank == 0: - logger.info("Solution diverged from exact_mix.") + logger.info("Solution diverged from exact soln.") return health_error @@ -279,7 +284,7 @@ def my_post_step(step, t, dt, state): def my_pre_step(step, t, dt, state): dv = None - exact_mix = None + exact = None pre_step_errors = False if logmgr: @@ -297,32 +302,31 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - exact_mix = initializer(x_vec=nodes, eos=eos, t=t) - local_health_error = my_health_check(state, dv, exact_mix) - health_errors = False + exact = initializer(x_vec=nodes, eos=eos, t=t) + health_errors = my_health_check(state, dv, exact) if comm is not None: - health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + health_errors = comm.allreduce(health_errors, op=MPI.LOR) if health_errors and rank == 0: logger.info("Fluid solution failed health check.") pre_step_errors = pre_step_errors or health_errors if do_restart: - my_write_restart(state, step, t) + my_write_restart(step=step, t=t, state=state) if do_viz: if dv is None: dv = eos.dependent_vars(state) - if exact_mix is None: - exact_mix = initializer(x_vec=nodes, eos=eos, t=t) - resid = state - exact_mix + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + resid = state - exact my_write_viz(cv=state, dv=dv, step=step, t=t, - exact_mix=exact_mix, resid=resid) + exact=exact, resid=resid) if do_status: - if exact_mix is None: - exact_mix = initializer(x_vec=nodes, eos=eos, t=t) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact_mix) + component_errors = compare_fluid_solutions(discr, state, exact) status_msg = ( "------- errors=" + ", ".join("%.3g" % en for en in component_errors)) @@ -330,7 +334,7 @@ def my_pre_step(step, t, dt, state): logger.info(status_msg) if pre_step_errors: - my_graceful_exit(cv=state, step=step, t=t, + my_graceful_exit(step=step, t=t, state=state, do_viz=(not do_viz), do_restart=(not do_restart), message="Error detected at prestep, exiting.") @@ -355,9 +359,9 @@ def my_pre_step(step, t, dt, state): final_dv = eos.dependent_vars(current_state) final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) final_resid = current_state - final_exact - my_write_viz(cv=current_state, dv=final_dv, exact_mix=final_exact, - resid=final_resid, step=current_step, t=current_t) - my_write_restart(current_state, current_step, current_t) + my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, + exact=final_exact, resid=final_resid) + my_write_restart(step=current_step, t=current_t, state=current_state) if logmgr: logmgr.close() diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 74c4b674e..18736ec68 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -198,26 +198,28 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) - def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, + message=None): if rank == 0: logger.info("Errors detected; attempting graceful exit.") if do_viz: - my_write_viz(cv, step, t) + my_write_viz(step=step, t=t, state=state) if do_restart: - my_write_restart(state=cv, step=step, t=t) + my_write_restart(step=step, t=t, state=state) if message is None: message = "Fatal simulation errors detected." raise RuntimeError(message) - def my_write_viz(cv, step, t, dv=None): - viz_fields = [("cv", cv)] - if dv is not None: - viz_fields.append(("dv", dv)) + def my_write_viz(step, t, state, dv=None): + if dv is None: + dv = eos.dependent_vars(state) + viz_fields = [("cv", state), + ("dv", dv)] from mirgecom.simutil import write_visfile write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) - def my_write_restart(state, step, t): + def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) rst_data = { "local_mesh": local_mesh, @@ -255,7 +257,7 @@ def my_post_step(step, t, dt, state): def my_pre_step(step, t, dt, state): dv = None pre_step_errors = False - print(f"{step=},{t=},{dt=}") + if logmgr: logmgr.tick_before() @@ -270,24 +272,23 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - local_health_error = my_health_check(dv, dt) - health_errors = False + health_errors = my_health_check(dv, dt) if comm is not None: - health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + health_errors = comm.allreduce(health_errors, op=MPI.LOR) if health_errors and rank == 0: logger.info("Fluid solution failed health check.") pre_step_errors = pre_step_errors or health_errors if do_restart: - my_write_restart(state, step, t) + my_write_restart(step=step, t=t, state=state) if do_viz: if dv is None: dv = eos.dependent_vars(state) - my_write_viz(cv=state, dv=dv, step=step, t=t) + my_write_viz(step=step, t=t, state=state, dv=dv) if pre_step_errors: - my_graceful_exit(cv=state, step=step, t=t, + my_graceful_exit(step=step, t=t, state=state, do_viz=(not do_viz), do_restart=(not do_restart), message="Error detected at prestep, exiting.") @@ -302,7 +303,7 @@ def my_pre_step(step, t, dt, state): finish_tol = 1e-16 if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(cv=current_state, step=current_step, t=current_t, + my_graceful_exit(step=current_step, t=current_t, state=current_state, do_viz=True, do_restart=True, message="Simulation timestepping did not complete.") @@ -310,8 +311,8 @@ def my_pre_step(step, t, dt, state): if rank == 0: logger.info("Checkpointing final state ...") final_dv = eos.dependent_vars(current_state) - my_write_viz(cv=current_state, dv=final_dv, step=current_step, t=current_t) - my_write_restart(current_state, current_step, current_t) + my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv) + my_write_restart(step=current_step, t=current_t, state=current_state) if logmgr: logmgr.close() diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 87e3bdb23..80507d92d 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -201,30 +201,34 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, cfl=current_cfl, eos=eos, t_final=t_final, constant_cfl=constant_cfl) - def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, + message=None): if rank == 0: logger.info("Errors detected; attempting graceful exit.") if do_viz: - my_write_viz(cv, step, t) + my_write_viz(step=step, t=t, state=state) if do_restart: - my_write_restart(state=cv, step=step, t=t) + my_write_restart(step=step, t=t, state=state) if message is None: message = "Fatal simulation errors detected." raise RuntimeError(message) - def my_write_viz(cv, step, t, dv=None, exact_lump=None, resid=None): - viz_fields = [("cv", cv)] - if dv is not None: - viz_fields.append(("dv", dv)) - if exact_lump is not None: - viz_fields.append(("exact", exact_lump)) - if resid is not None: - viz_fields.append(("residual", resid)) + def my_write_viz(step, t, state, dv=None, exact=None, resid=None): + if dv is None: + dv = eos.dependent_vars(state) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + if resid is None: + resid = state - exact + viz_fields = [("cv", state), + ("dv", dv), + ("exact", exact), + ("resid", resid)] from mirgecom.simutil import write_visfile write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) - def my_write_restart(state, step, t): + def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) rst_data = { "local_mesh": local_mesh, @@ -237,7 +241,7 @@ def my_write_restart(state, step, t): from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) - def my_health_check(state, dv, exact_lump): + def my_health_check(state, dv, exact): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ @@ -246,12 +250,12 @@ def my_health_check(state, dv, exact_lump): logger.info(f"{rank=}: Invalid pressure data found.") from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact_lump) + component_errors = compare_fluid_solutions(discr, state, exact) exittol = .09 if max(component_errors) > exittol: health_error = True if rank == 0: - logger.info("Solution diverged from exact_lump.") + logger.info("Solution diverged from exact soln.") return health_error @@ -266,7 +270,7 @@ def my_post_step(step, t, dt, state): def my_pre_step(step, t, dt, state): dv = None - exact_lump = None + exact = None pre_step_errors = False if logmgr: @@ -284,32 +288,31 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - exact_lump = initializer(x_vec=nodes, eos=eos, t=t) - local_health_error = my_health_check(state, dv, exact_lump) - health_errors = False + exact = initializer(x_vec=nodes, eos=eos, t=t) + health_errors = my_health_check(state, dv, exact) if comm is not None: - health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + health_errors = comm.allreduce(health_errors, op=MPI.LOR) if health_errors and rank == 0: logger.info("Fluid solution failed health check.") pre_step_errors = pre_step_errors or health_errors if do_restart: - my_write_restart(state, step, t) + my_write_restart(step=step, t=t, state=state) if do_viz: if dv is None: dv = eos.dependent_vars(state) - if exact_lump is None: - exact_lump = initializer(x_vec=nodes, eos=eos, t=t) - resid = state - exact_lump - my_write_viz(cv=state, dv=dv, step=step, t=t, - exact_lump=exact_lump, resid=resid) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + resid = state - exact + my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, + resid=resid) if do_status: - if exact_lump is None: - exact_lump = initializer(x_vec=nodes, eos=eos, t=t) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact_lump) + component_errors = compare_fluid_solutions(discr, state, exact) status_msg = ( "------- errors=" + ", ".join("%.3g" % en for en in component_errors)) @@ -317,7 +320,7 @@ def my_pre_step(step, t, dt, state): logger.info(status_msg) if pre_step_errors: - my_graceful_exit(cv=state, step=step, t=t, + my_graceful_exit(step=step, t=t, state=state, do_viz=(not do_viz), do_restart=(not do_restart), message="Error detected at prestep, exiting.") @@ -336,7 +339,7 @@ def my_rhs(t, state): finish_tol = 1e-16 if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(cv=current_state, step=current_step, t=current_t, + my_graceful_exit(step=current_step, t=current_t, state=current_state, do_viz=True, do_restart=True, message="Simulation timestepping did not complete.") @@ -347,9 +350,9 @@ def my_rhs(t, state): final_dv = eos.dependent_vars(current_state) final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) final_resid = current_state - final_exact - my_write_viz(cv=current_state, dv=final_dv, exact_lump=final_exact, - resid=final_resid, step=current_step, t=current_t) - my_write_restart(current_state, current_step, current_t) + my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, + exact=final_exact, resid=final_resid) + my_write_restart(step=current_step, t=current_t, state=current_state) if logmgr: logmgr.close() diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 286e6b07e..2434be2c7 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -182,25 +182,29 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, if rank == 0: logger.info(init_message) - def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, + message=None): if rank == 0: logger.info("Errors detected; attempting graceful exit.") if do_viz: - my_write_viz(cv, step, t) + my_write_viz(step=step, t=t, state=state) if do_restart: - my_write_restart(state=cv, step=step, t=t) + my_write_restart(step=step, t=t, state=state) if message is None: message = "Fatal simulation errors detected." raise RuntimeError(message) - def my_write_viz(cv, step, t, dv=None, exact=None, resid=None): - viz_fields = [("cv", cv)] - if dv is not None: - viz_fields.append(("dv", dv)) - if exact is not None: - viz_fields.append(("exact_soln", exact)) - if resid is not None: - viz_fields.append(("residual", resid)) + def my_write_viz(step, t, state, dv=None, exact=None, resid=None): + if dv is None: + dv = eos.dependent_vars(state) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + if resid is None: + resid = state - exact + viz_fields = [("cv", state), + ("dv", dv), + ("exact", exact), + ("residual", resid)] from mirgecom.simutil import write_visfile write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) @@ -266,16 +270,15 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) exact = initializer(x_vec=nodes, eos=eos, t=t) - local_health_error = my_health_check(state, dv, exact) - health_errors = False + health_errors = my_health_check(state, dv, exact) if comm is not None: - health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + health_errors = comm.allreduce(health_errors, op=MPI.LOR) if health_errors and rank == 0: logger.info("Fluid solution failed health check.") pre_step_errors = pre_step_errors or health_errors if do_restart: - my_write_restart(state, step, t) + my_write_restart(step=step, t=t, state=state) if do_viz: if dv is None: @@ -283,8 +286,8 @@ def my_pre_step(step, t, dt, state): if exact is None: exact = initializer(x_vec=nodes, eos=eos, t=t) resid = state - exact - my_write_viz(cv=state, dv=dv, step=step, t=t, - exact=exact, resid=resid) + my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, + resid=resid) if do_status: if exact is None: @@ -298,7 +301,7 @@ def my_pre_step(step, t, dt, state): logger.info(status_msg) if pre_step_errors: - my_graceful_exit(cv=state, step=step, t=t, + my_graceful_exit(step=step, t=t, state=state, do_viz=(not do_viz), do_restart=(not do_restart), message="Error detected at prestep, exiting.") @@ -321,7 +324,7 @@ def my_rhs(t, state): finish_tol = 1e-16 if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(cv=current_state, step=current_step, t=current_t, + my_graceful_exit(step=current_step, t=current_t, state=current_state, do_viz=True, do_restart=True, message="Simulation timestepping did not complete.") @@ -332,9 +335,9 @@ def my_rhs(t, state): final_dv = eos.dependent_vars(current_state) final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) final_resid = current_state - final_exact - my_write_viz(cv=current_state, dv=final_dv, exact=final_exact, - resid=final_resid, step=current_step, t=current_t) - my_write_restart(current_state, current_step, current_t) + my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, + exact=final_exact, resid=final_resid) + my_write_restart(step=current_step, t=current_t, state=current_state) if logmgr: logmgr.close() diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index bb170b7d5..db34a3e5d 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -202,30 +202,34 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, if rank == 0: logger.info(init_message) - def my_graceful_exit(cv, step, t, do_viz=False, do_restart=False, message=None): + def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, + message=None): if rank == 0: logger.info("Errors detected; attempting graceful exit.") if do_viz: - my_write_viz(cv, step, t) + my_write_viz(step=step, t=t, state=state) if do_restart: - my_write_restart(state=cv, step=step, t=t) + my_write_restart(step=step, t=t, state=state) if message is None: message = "Fatal simulation errors detected." raise RuntimeError(message) - def my_write_viz(cv, step, t, dv=None, exact=None, resid=None): - viz_fields = [("cv", cv)] - if dv is not None: - viz_fields.append(("dv", dv)) - if exact is not None: - viz_fields.append(("exact_soln", exact)) - if resid is not None: - viz_fields.append(("residual", resid)) + def my_write_viz(step, t, state, dv=None, exact=None, resid=None): + if dv is None: + dv = eos.dependent_vars(state) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + if resid is None: + resid = state - exact + viz_fields = [("cv", state), + ("dv", dv), + ("exact", exact), + ("residual", resid)] from mirgecom.simutil import write_visfile write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) - def my_write_restart(state, step, t): + def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) rst_data = { "local_mesh": local_mesh, @@ -285,16 +289,15 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) exact = initializer(x_vec=nodes, eos=eos, t=t) - local_health_error = my_health_check(state, dv, exact) - health_errors = False + health_errors = my_health_check(state, dv, exact) if comm is not None: - health_errors = comm.allreduce(local_health_error, op=MPI.LOR) + health_errors = comm.allreduce(health_errors, op=MPI.LOR) if health_errors and rank == 0: logger.info("Fluid solution failed health check.") pre_step_errors = pre_step_errors or health_errors if do_restart: - my_write_restart(state, step, t) + my_write_restart(step=step, t=t, state=state) if do_viz: if dv is None: @@ -302,8 +305,8 @@ def my_pre_step(step, t, dt, state): if exact is None: exact = initializer(x_vec=nodes, eos=eos, t=t) resid = state - exact - my_write_viz(cv=state, dv=dv, step=step, t=t, - exact=exact, resid=resid) + my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, + resid=resid) if do_status: if exact is None: @@ -317,7 +320,7 @@ def my_pre_step(step, t, dt, state): logger.info(status_msg) if pre_step_errors: - my_graceful_exit(cv=state, step=step, t=t, + my_graceful_exit(step=step, t=t, state=state, do_viz=(not do_viz), do_restart=(not do_restart), message="Error detected at prestep, exiting.") @@ -340,7 +343,7 @@ def my_rhs(t, state): finish_tol = 1e-16 if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(cv=current_state, step=current_step, t=current_t, + my_graceful_exit(step=current_step, t=current_t, state=current_state, do_viz=True, do_restart=True, message="Simulation timestepping did not complete.") @@ -351,9 +354,9 @@ def my_rhs(t, state): final_dv = eos.dependent_vars(current_state) final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) final_resid = current_state - final_exact - my_write_viz(cv=current_state, dv=final_dv, exact=final_exact, - resid=final_resid, step=current_step, t=current_t) - my_write_restart(current_state, current_step, current_t) + my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, + exact=final_exact, resid=final_resid) + my_write_restart(step=current_step, t=current_t, state=current_state) if logmgr: logmgr.close() From 7d6c988ff9e226d1a6cc02a477eef13864979473 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 8 Jul 2021 10:38:22 -0500 Subject: [PATCH 228/385] Correct some dumb issues with the examples. --- examples/autoignition-mpi.py | 2 +- examples/lump-mpi.py | 1 - examples/mixture-mpi.py | 3 +-- examples/pulse-mpi.py | 1 - examples/scalar-lump-mpi.py | 3 +-- examples/sod-mpi.py | 1 - 6 files changed, 3 insertions(+), 8 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index ff4c56a2e..c9a7689bf 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -437,7 +437,7 @@ def my_pre_step(step, t, dt, state): if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) - use_profiling = True + use_profiling = False use_logging = True use_leap = False casename = "autoignition" diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index de6f723b1..76937c734 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -111,7 +111,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, nhealth = 1 nrestart = 10 nviz = 1 - rank = 0 current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index f61e740be..91aa2b198 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -110,7 +110,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, nhealth = 1 nrestart = 5 nviz = 1 - rank = 0 current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder @@ -319,7 +318,7 @@ def my_pre_step(step, t, dt, state): if exact is None: exact = initializer(x_vec=nodes, eos=eos, t=t) resid = state - exact - my_write_viz(cv=state, dv=dv, step=step, t=t, + my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, resid=resid) if do_status: diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 18736ec68..5e07d8779 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -118,7 +118,6 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, nrestart = 5 nviz = 10 nhealth = 1 - rank = 0 current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 80507d92d..e12b507eb 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -97,7 +97,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, dim = 3 nel_1d = 16 order = 3 - t_final = 0.01 + t_final = 0.005 current_cfl = 1.0 current_dt = .001 current_t = 0 @@ -106,7 +106,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, nrestart = 5 nviz = 1 nhealth = 1 - rank = 0 current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 2434be2c7..6de59f751 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -109,7 +109,6 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, nrestart = 5 nviz = 10 nhealth = 10 - rank = 0 current_step = 0 if use_leap: from leap.rk import RK4MethodBuilder From ab06ff7d8a2edf5f8507688567164952e12bf098 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 8 Jul 2021 10:52:43 -0500 Subject: [PATCH 229/385] Correct errant call-sites to new utility signatures. --- examples/autoignition-mpi.py | 4 ++-- examples/lump-mpi.py | 4 ++-- examples/mixture-mpi.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index c9a7689bf..74ed10ba0 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -403,7 +403,7 @@ def my_pre_step(step, t, dt, state): production_rates=production_rates) if pre_step_errors: - my_graceful_exit(cv=state, step=step, t=t, + my_graceful_exit(step=step, t=t, state=state, do_viz=(not do_viz), do_restart=(not do_restart), message="Error detected at prestep, exiting.") @@ -418,7 +418,7 @@ def my_pre_step(step, t, dt, state): finish_tol = 1e-16 if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(cv=current_state, step=current_step, t=current_t, + my_graceful_exit(step=current_step, t=current_t, state=current_state, do_viz=True, do_restart=True, message="Simulation timestepping did not complete.") diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 76937c734..ea1375ed9 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -312,7 +312,7 @@ def my_pre_step(step, t, dt, state): logger.info(status_msg) if pre_step_errors: - my_graceful_exit(cv=state, step=step, t=t, + my_graceful_exit(step=step, t=t, state=state, do_viz=(not do_viz), do_restart=(not do_restart), message="Error detected at prestep, exiting.") @@ -327,7 +327,7 @@ def my_pre_step(step, t, dt, state): finish_tol = 1e-16 if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(cv=current_state, step=current_step, t=current_t, + my_graceful_exit(step=current_step, t=current_t, state=current_state, do_viz=True, do_restart=True, message="Simulation timestepping did not complete.") diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 91aa2b198..b433da1c0 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -348,7 +348,7 @@ def my_pre_step(step, t, dt, state): finish_tol = 1e-16 if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(cv=current_state, step=current_step, t=current_t, + my_graceful_exit(step=current_step, t=current_t, state=current_state, do_viz=True, do_restart=True, message="Simulation timestepping did not complete.") From 9c631364d56002955260a94b7cd5b95568a41564 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 8 Jul 2021 11:04:50 -0500 Subject: [PATCH 230/385] Call logmgr_set_time on restart. --- examples/autoignition-mpi.py | 3 +++ examples/lump-mpi.py | 3 +++ examples/mixture-mpi.py | 3 +++ examples/pulse-mpi.py | 3 +++ examples/scalar-lump-mpi.py | 3 +++ examples/sod-mpi.py | 3 +++ examples/vortex-mpi.py | 3 +++ 7 files changed, 21 insertions(+) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 74ed10ba0..74487e0a6 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -257,6 +257,9 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, current_t = restart_data["t"] current_step = rst_step current_state = restart_data["state"] + if logmgr: + from logging_quantities import logmgr_set_time + logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 current_state = initializer(eos=eos, x_vec=nodes, t=0) diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index ea1375ed9..7a1663ab5 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -168,6 +168,9 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, current_t = restart_data["t"] current_step = rst_step current_state = restart_data["state"] + if logmgr: + from logging_quantities import logmgr_set_time + logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 current_state = initializer(nodes) diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index b433da1c0..bb27da3e2 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -188,6 +188,9 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, current_t = restart_data["t"] current_step = rst_step current_state = restart_data["state"] + if logmgr: + from logging_quantities import logmgr_set_time + logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 current_state = initializer(x_vec=nodes, eos=eos) diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 5e07d8779..8fdd5ee29 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -175,6 +175,9 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, current_t = restart_data["t"] current_step = rst_step current_state = restart_data["state"] + if logmgr: + from logging_quantities import logmgr_set_time + logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 current_state = acoustic_pulse(x_vec=nodes, cv=uniform_state, eos=eos) diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index e12b507eb..aac58bd6a 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -179,6 +179,9 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, current_t = restart_data["t"] current_step = rst_step current_state = restart_data["state"] + if logmgr: + from logging_quantities import logmgr_set_time + logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 current_state = initializer(nodes) diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 6de59f751..ad48851b0 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -163,6 +163,9 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, current_t = restart_data["t"] current_step = rst_step current_state = restart_data["state"] + if logmgr: + from logging_quantities import logmgr_set_time + logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 current_state = initializer(nodes) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index db34a3e5d..e86f60191 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -184,6 +184,9 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, current_t = restart_data["t"] current_step = rst_step current_state = restart_data["state"] + if logmgr: + from logging_quantities import logmgr_set_time + logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 current_state = initializer(nodes) From 1b4df75407f3e6e53cd347194497b99df687bada Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 8 Jul 2021 11:23:10 -0500 Subject: [PATCH 231/385] Deprecate logmgr, dim, eos args to stepper. --- mirgecom/steppers.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 765d1872e..6bc8aac2c 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -296,6 +296,13 @@ def advance_state(rhs, timestepper, state, t_final, # First, check if we have leap. import sys leap_timestepper = False + + if ((logmgr is not None) or (dim is not None) or (eos is not None)): + from warnings import warn + warn("Passing logmgr, dim, or eos into the stepper is a deprecated stepper " + "signature. See the examples for the current and preferred usage.", + DeprecationWarning, stacklevel=2) + if "leap" in sys.modules: # The timestepper can still either be a leap method generator # or a user-passed function. From c857e067f352a8d4f4eb0ad3fa192d14c17d2ab4 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 8 Jul 2021 11:29:25 -0500 Subject: [PATCH 232/385] Correct mistake in input restart file naming. --- examples/autoignition-mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 74487e0a6..35181ff8a 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -140,7 +140,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) + rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_fname) From bdd4d90cec9c845cab82ad91f6e167c68806eefb Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 8 Jul 2021 11:38:38 -0500 Subject: [PATCH 233/385] Correct mistake in input restart file naming. --- examples/lump-mpi.py | 2 +- examples/mixture-mpi.py | 2 +- examples/pulse-mpi.py | 2 +- examples/scalar-lump-mpi.py | 2 +- examples/sod-mpi.py | 2 +- examples/vortex-mpi.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 7a1663ab5..835553dab 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -123,7 +123,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) + rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_fname) diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index bb27da3e2..c6c8e9941 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -122,7 +122,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) + rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_fname) diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 8fdd5ee29..3229700d6 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -130,7 +130,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) + rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_fname) diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index aac58bd6a..f25f5d71c 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -134,7 +134,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) + rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_fname) diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index ad48851b0..0406c6f87 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -121,7 +121,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) + rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_fname) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index e86f60191..8908af79b 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -129,7 +129,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=casename, step=rst_step, rank=rank) + rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_fname) From 1a7e8e3cec7ec3efbb0b25ebf1d9c37aa914e1cf Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 8 Jul 2021 12:00:49 -0500 Subject: [PATCH 234/385] Correct import location for logmgr_set_time --- examples/autoignition-mpi.py | 2 +- examples/lump-mpi.py | 2 +- examples/mixture-mpi.py | 2 +- examples/pulse-mpi.py | 2 +- examples/scalar-lump-mpi.py | 2 +- examples/sod-mpi.py | 2 +- examples/vortex-mpi.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 35181ff8a..2bafb8e28 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -258,7 +258,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, current_step = rst_step current_state = restart_data["state"] if logmgr: - from logging_quantities import logmgr_set_time + from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 835553dab..ce512a059 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -169,7 +169,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, current_step = rst_step current_state = restart_data["state"] if logmgr: - from logging_quantities import logmgr_set_time + from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index c6c8e9941..8069fc43c 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -189,7 +189,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, current_step = rst_step current_state = restart_data["state"] if logmgr: - from logging_quantities import logmgr_set_time + from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 3229700d6..bec26dd74 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -176,7 +176,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, current_step = rst_step current_state = restart_data["state"] if logmgr: - from logging_quantities import logmgr_set_time + from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index f25f5d71c..0980a528e 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -180,7 +180,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, current_step = rst_step current_state = restart_data["state"] if logmgr: - from logging_quantities import logmgr_set_time + from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 0406c6f87..fc9782f0a 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -164,7 +164,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, current_step = rst_step current_state = restart_data["state"] if logmgr: - from logging_quantities import logmgr_set_time + from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 8908af79b..06c1a94ae 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -185,7 +185,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, current_step = rst_step current_state = restart_data["state"] if logmgr: - from logging_quantities import logmgr_set_time + from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 From 6a7d20c4f93c2e44168e6a56ad3064270056ae21 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 8 Jul 2021 13:36:08 -0500 Subject: [PATCH 235/385] Document dt, deprecate get_timestep arg --- mirgecom/steppers.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 6bc8aac2c..7a60f5600 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -61,6 +61,8 @@ def _advance_state_stepper_func(rhs, timestepper, Simulated time at which to stop t: float Time at which to start + dt: float + Initial timestep size to use, optional if dt is adaptive istep: int Step number from which to start pre_step_callback @@ -140,6 +142,8 @@ def _advance_state_leap(rhs, timestepper, state, State id (required input for leap method generation) t: float Time at which to start + dt: float + Initial timestep size to use, optional if dt is adaptive istep: int Step number from which to start pre_step_callback @@ -272,6 +276,8 @@ def advance_state(rhs, timestepper, state, t_final, Simulated time at which to stop t: float Time at which to start + dt: float + Initial timestep size to use, optional if dt is adaptive istep: int Step number from which to start pre_step_callback @@ -303,6 +309,13 @@ def advance_state(rhs, timestepper, state, t_final, "signature. See the examples for the current and preferred usage.", DeprecationWarning, stacklevel=2) + if get_timestep is not None: + from warnings import warn + warn("Passing the get_timestep function into the stepper is deprecated. " + "Users should use the dt argument for constant timestep, and " + "perform any dt modification in the {pre,post}-step callbacks.", + DeprecationWarning, stacklevel=2) + if "leap" in sys.modules: # The timestepper can still either be a leap method generator # or a user-passed function. From ea98b59a44da492ab687e63ecc03c1c96b20c721 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 8 Jul 2021 20:24:52 -0500 Subject: [PATCH 236/385] Update examples to use exceptions to clean up error handling, evict get_timestep, add order to restart data. --- examples/autoignition-mpi.py | 120 +++++++++++++-------------- examples/lump-mpi.py | 145 ++++++++++++++++----------------- examples/mixture-mpi.py | 146 ++++++++++++++++----------------- examples/pulse-mpi.py | 119 +++++++++++++-------------- examples/scalar-lump-mpi.py | 151 +++++++++++++++++----------------- examples/sod-mpi.py | 153 +++++++++++++++++------------------ examples/vortex-mpi.py | 145 ++++++++++++++++----------------- 7 files changed, 466 insertions(+), 513 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 2bafb8e28..71941774f 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -43,7 +43,6 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( - inviscid_sim_timestep, generate_and_distribute_mesh, write_visfile ) @@ -70,6 +69,18 @@ logger = logging.getLogger(__name__) +class MyError(Exception): + """Simple exception to kill the simulation.""" + + pass + + +class HealthCheckError(MyError): + """Simple exception to indicate a health check error.""" + + pass + + @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=False, use_leap=False, use_profiling=False, casename="autoignition", @@ -297,22 +308,6 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, f" {eq_pressure=}, {eq_temperature=}," f" {eq_density=}, {eq_mass_fractions=}") - get_timestep = partial(inviscid_sim_timestep, discr=discr, - cfl=current_cfl, eos=eos, - t_final=t_final, constant_cfl=constant_cfl) - - def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, - message=None): - if rank == 0: - logger.info("Errors detected; attempting graceful exit.") - if do_viz: - my_write_viz(step=step, t=t, state=state) - if do_restart: - my_write_restart(step=step, t=t, state=state) - if message is None: - message = "Fatal simulation errors detected." - raise RuntimeError(message) - def my_write_viz(step, t, state, dv=None, production_rates=None): if dv is None: dv = eos.dependent_vars(state) @@ -331,13 +326,14 @@ def my_write_restart(step, t, state): "state": state, "t": t, "step": step, + "order": order, "global_nelements": global_nelements, "num_parts": nproc } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) - def my_health_check(dv, dt): + def my_health_check(dv): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ @@ -349,11 +345,6 @@ def my_health_check(dv, dt): health_error = True logger.info(f"{rank=}: Invalid temperature data found.") - if dt < 0: - health_error = True - if rank == 0: - logger.info("Global DT is negative!") - return health_error def my_rhs(t, state): @@ -371,44 +362,49 @@ def my_post_step(step, t, dt, state): return state, dt def my_pre_step(step, t, dt, state): - dv = None - pre_step_errors = False + try: + dv = None - if logmgr: - logmgr.tick_before() - - from mirgecom.simutil import check_step - do_viz = check_step(step=step, interval=nviz) - do_restart = check_step(step=step, interval=nrestart) - do_health = check_step(step=step, interval=nhealth) + if logmgr: + logmgr.tick_before() - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False + from mirgecom.simutil import check_step + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) - if do_health: - dv = eos.dependent_vars(state) - health_errors = my_health_check(dv, dt) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) - if health_errors and rank == 0: - logger.info("Fluid solution failed health check.") - pre_step_errors = pre_step_errors or health_errors - - if do_restart: - my_write_restart(step=step, t=t, state=state) + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False - if do_viz: - production_rates = eos.get_production_rates(state) - if dv is None: + if do_health: dv = eos.dependent_vars(state) - my_write_viz(step=step, t=t, state=state, dv=dv, - production_rates=production_rates) - - if pre_step_errors: - my_graceful_exit(step=step, t=t, state=state, - do_viz=(not do_viz), do_restart=(not do_restart), - message="Error detected at prestep, exiting.") + health_errors = my_health_check(dv) + if comm is not None: + health_errors = comm.allreduce(health_errors, op=MPI.LOR) + if step > 5: + health_errors = True + if health_errors: + if rank == 0: + logger.info("Fluid solution failed health check.") + raise HealthCheckError() + + if do_restart: + my_write_restart(step=step, t=t, state=state) + + if do_viz: + production_rates = eos.get_production_rates(state) + if dv is None: + dv = eos.dependent_vars(state) + my_write_viz(step=step, t=t, state=state, dv=dv, + production_rates=production_rates) + + except MyError: + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + my_write_viz(step=step, t=t, state=state) + my_write_restart(step=step, t=t, state=state) + raise return state, dt @@ -416,14 +412,7 @@ def my_pre_step(step, t, dt, state): advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final, eos=eos, dim=dim) - - finish_tol = 1e-16 - if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(step=current_step, t=current_t, state=current_state, - do_viz=True, do_restart=True, - message="Simulation timestepping did not complete.") + state=current_state, t=current_t, t_final=t_final) # Dump the final data final_dv = eos.dependent_vars(current_state) @@ -437,6 +426,9 @@ def my_pre_step(step, t, dt, state): elif use_profiling: print(actx.tabulate_profiling_data()) + finish_tol = 1e-16 + assert (current_t - t_final) > finish_tol + if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index ce512a059..7f5cd46cc 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -37,10 +37,7 @@ from mirgecom.euler import euler_operator -from mirgecom.simutil import ( - inviscid_sim_timestep, - generate_and_distribute_mesh -) +from mirgecom.simutil import generate_and_distribute_mesh from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -64,6 +61,18 @@ logger = logging.getLogger(__name__) +class MyError(Exception): + """Simple exception to kill the simulation.""" + + pass + + +class HealthCheckError(MyError): + """Simple exception to indicate a health check error.""" + + pass + + @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_leap=False, use_profiling=False, rst_step=None, rst_name=None, @@ -188,22 +197,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, if rank == 0: logger.info(init_message) - get_timestep = partial(inviscid_sim_timestep, discr=discr, - cfl=current_cfl, eos=eos, - t_final=t_final, constant_cfl=constant_cfl) - - def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, - message=None): - if rank == 0: - logger.info("Errors detected; attempting graceful exit.") - if do_viz: - my_write_viz(step=step, t=t, state=state) - if do_restart: - my_write_restart(step=step, t=t, state=state) - if message is None: - message = "Fatal simulation errors detected." - raise RuntimeError(message) - def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: dv = eos.dependent_vars(state) @@ -226,6 +219,7 @@ def my_write_restart(state, step, t): "state": state, "t": t, "step": step, + "order": order, "global_nelements": global_nelements, "num_parts": nparts } @@ -264,60 +258,63 @@ def my_post_step(step, t, dt, state): return state, dt def my_pre_step(step, t, dt, state): - dv = None - exact = None - pre_step_errors = False + try: + dv = None + exact = None - if logmgr: - logmgr.tick_before() - - from mirgecom.simutil import check_step - do_viz = check_step(step=step, interval=nviz) - do_restart = check_step(step=step, interval=nrestart) - do_health = check_step(step=step, interval=nhealth) - do_status = check_step(step=step, interval=nstatus) + if logmgr: + logmgr.tick_before() - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False + from mirgecom.simutil import check_step + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) + do_status = check_step(step=step, interval=nstatus) - if do_health: - dv = eos.dependent_vars(state) - exact = initializer(x_vec=nodes, eos=eos, t=t) - health_errors = my_health_check(dv=dv, state=state, exact=exact) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) - if health_errors and rank == 0: - logger.info("Fluid solution failed health check.") - pre_step_errors = pre_step_errors or health_errors - - if do_restart: - my_write_restart(step=step, t=t, state=state) - - if do_viz: - if dv is None: + if do_health: dv = eos.dependent_vars(state) - if exact is None: exact = initializer(x_vec=nodes, eos=eos, t=t) - resid = state - exact - my_write_viz(step=step, t=t, dv=dv, state=state, exact=exact, - resid=resid) - - if do_status: - if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) - from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact) - status_msg = ( - "------- errors=" - + ", ".join("%.3g" % en for en in component_errors)) + health_errors = my_health_check(dv=dv, state=state, exact=exact) + if comm is not None: + health_errors = comm.allreduce(health_errors, op=MPI.LOR) + if health_errors: + if rank == 0: + logger.info("Fluid solution failed health check.") + raise HealthCheckError() + + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False + + if do_restart: + my_write_restart(step=step, t=t, state=state) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + resid = state - exact + my_write_viz(step=step, t=t, dv=dv, state=state, exact=exact, + resid=resid) + + if do_status: + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + status_msg = ( + "------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + if rank == 0: + logger.info(status_msg) + + except MyError: if rank == 0: - logger.info(status_msg) - - if pre_step_errors: - my_graceful_exit(step=step, t=t, state=state, - do_viz=(not do_viz), do_restart=(not do_restart), - message="Error detected at prestep, exiting.") + logger.info("Errors detected; attempting graceful exit.") + my_write_viz(step=step, t=t, state=state) + my_write_restart(step=step, t=t, state=state) + raise return state, dt @@ -325,14 +322,7 @@ def my_pre_step(step, t, dt, state): advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final, eos=eos, dim=dim) - - finish_tol = 1e-16 - if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(step=current_step, t=current_t, state=current_state, - do_viz=True, do_restart=True, - message="Simulation timestepping did not complete.") + state=current_state, t=current_t, t_final=t_final) # Dump the final data if rank == 0: @@ -350,6 +340,9 @@ def my_pre_step(step, t, dt, state): elif use_profiling: print(actx.tabulate_profiling_data()) + finish_tol = 1e-16 + assert (current_t - t_final) > finish_tol + if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 8069fc43c..ffc3005dc 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -37,10 +37,7 @@ from mirgecom.euler import euler_operator -from mirgecom.simutil import ( - inviscid_sim_timestep, - generate_and_distribute_mesh -) +from mirgecom.simutil import generate_and_distribute_mesh from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -67,6 +64,18 @@ logger = logging.getLogger(__name__) +class MyError(Exception): + """Simple exception to kill the simulation.""" + + pass + + +class HealthCheckError(MyError): + """Simple exception to indicate a health check error.""" + + pass + + @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_leap=False, use_profiling=False, rst_step=None, rst_name=None, @@ -208,22 +217,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, if rank == 0: logger.info(init_message) - get_timestep = partial(inviscid_sim_timestep, discr=discr, - cfl=current_cfl, eos=eos, - t_final=t_final, constant_cfl=constant_cfl) - - def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, - message=None): - if rank == 0: - logger.info("Errors detected; attempting graceful exit.") - if do_viz: - my_write_viz(step=step, t=t, state=state) - if do_restart: - my_write_restart(step=step, t=t, state=state) - if message is None: - message = "Fatal simulation errors detected." - raise RuntimeError(message) - def my_write_viz(step, t, state, dv=None, exact=None, resid=None): viz_fields = [("cv", state)] if dv is None: @@ -247,6 +240,7 @@ def my_write_restart(step, t, state): "state": state, "t": t, "step": step, + "order": order, "global_nelements": global_nelements, "num_parts": nparts } @@ -285,60 +279,63 @@ def my_post_step(step, t, dt, state): return state, dt def my_pre_step(step, t, dt, state): - dv = None - exact = None - pre_step_errors = False + try: + dv = None + exact = None - if logmgr: - logmgr.tick_before() - - from mirgecom.simutil import check_step - do_viz = check_step(step=step, interval=nviz) - do_restart = check_step(step=step, interval=nrestart) - do_health = check_step(step=step, interval=nhealth) - do_status = check_step(step=step, interval=nstatus) + if logmgr: + logmgr.tick_before() - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False + from mirgecom.simutil import check_step + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) + do_status = check_step(step=step, interval=nstatus) - if do_health: - dv = eos.dependent_vars(state) - exact = initializer(x_vec=nodes, eos=eos, t=t) - health_errors = my_health_check(state, dv, exact) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) - if health_errors and rank == 0: - logger.info("Fluid solution failed health check.") - pre_step_errors = pre_step_errors or health_errors - - if do_restart: - my_write_restart(step=step, t=t, state=state) - - if do_viz: - if dv is None: + if do_health: dv = eos.dependent_vars(state) - if exact is None: exact = initializer(x_vec=nodes, eos=eos, t=t) - resid = state - exact - my_write_viz(step=step, t=t, state=state, dv=dv, - exact=exact, resid=resid) - - if do_status: - if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) - from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact) - status_msg = ( - "------- errors=" - + ", ".join("%.3g" % en for en in component_errors)) + health_errors = my_health_check(state, dv, exact) + if comm is not None: + health_errors = comm.allreduce(health_errors, op=MPI.LOR) + if health_errors: + if rank == 0: + logger.info("Fluid solution failed health check.") + raise HealthCheckError() + + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False + + if do_restart: + my_write_restart(step=step, t=t, state=state) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + resid = state - exact + my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, + resid=resid) + + if do_status: + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + status_msg = ( + "------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + if rank == 0: + logger.info(status_msg) + + except MyError: if rank == 0: - logger.info(status_msg) - - if pre_step_errors: - my_graceful_exit(step=step, t=t, state=state, - do_viz=(not do_viz), do_restart=(not do_restart), - message="Error detected at prestep, exiting.") + logger.info("Errors detected; attempting graceful exit.") + my_write_viz(step=step, t=t, state=state) + my_write_restart(step=step, t=t, state=state) + raise return state, dt @@ -346,14 +343,8 @@ def my_pre_step(step, t, dt, state): advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final, eos=eos, dim=dim) - - finish_tol = 1e-16 - if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(step=current_step, t=current_t, state=current_state, - do_viz=True, do_restart=True, - message="Simulation timestepping did not complete.") + state=current_state, t=current_t, t_final=t_final, eos=eos, + dim=dim) # Dump the final data if rank == 0: @@ -370,6 +361,9 @@ def my_pre_step(step, t, dt, state): elif use_profiling: print(actx.tabulate_profiling_data()) + finish_tol = 1e-16 + assert (current_t - t_final) > finish_tol + if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index bec26dd74..8875979d8 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -38,10 +38,7 @@ from grudge.shortcuts import make_visualizer from mirgecom.euler import euler_operator -from mirgecom.simutil import ( - inviscid_sim_timestep, - generate_and_distribute_mesh -) +from mirgecom.simutil import generate_and_distribute_mesh from mirgecom.io import make_init_message from mirgecom.integrators import rk4_step @@ -70,6 +67,18 @@ logger = logging.getLogger(__name__) +class MyError(Exception): + """Simple exception to kill the simulation.""" + + pass + + +class HealthCheckError(MyError): + """Simple exception to indicate a health check error.""" + + pass + + @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename="pulse", @@ -196,22 +205,6 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, if rank == 0: logger.info(init_message) - get_timestep = partial(inviscid_sim_timestep, discr=discr, - cfl=current_cfl, eos=eos, t_final=t_final, - constant_cfl=constant_cfl) - - def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, - message=None): - if rank == 0: - logger.info("Errors detected; attempting graceful exit.") - if do_viz: - my_write_viz(step=step, t=t, state=state) - if do_restart: - my_write_restart(step=step, t=t, state=state) - if message is None: - message = "Fatal simulation errors detected." - raise RuntimeError(message) - def my_write_viz(step, t, state, dv=None): if dv is None: dv = eos.dependent_vars(state) @@ -228,17 +221,18 @@ def my_write_restart(step, t, state): "state": state, "t": t, "step": step, + "order": order, "global_nelements": global_nelements, "num_parts": num_parts } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) - def my_health_check(dv, dt): + def my_health_check(pressure): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure, .8, 1.5): + if check_naninf_local(discr, "vol", pressure) \ + or check_range_local(discr, "vol", pressure, .8, 1.5): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") return health_error @@ -257,42 +251,45 @@ def my_post_step(step, t, dt, state): return state, dt def my_pre_step(step, t, dt, state): - dv = None - pre_step_errors = False + try: + dv = None - if logmgr: - logmgr.tick_before() + if logmgr: + logmgr.tick_before() - from mirgecom.simutil import check_step - do_viz = check_step(step=step, interval=nviz) - do_restart = check_step(step=step, interval=nrestart) - do_health = check_step(step=step, interval=nhealth) + from mirgecom.simutil import check_step + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False - - if do_health: - dv = eos.dependent_vars(state) - health_errors = my_health_check(dv, dt) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) - if health_errors and rank == 0: - logger.info("Fluid solution failed health check.") - pre_step_errors = pre_step_errors or health_errors - - if do_restart: - my_write_restart(step=step, t=t, state=state) - - if do_viz: - if dv is None: + if do_health: dv = eos.dependent_vars(state) - my_write_viz(step=step, t=t, state=state, dv=dv) - - if pre_step_errors: - my_graceful_exit(step=step, t=t, state=state, - do_viz=(not do_viz), do_restart=(not do_restart), - message="Error detected at prestep, exiting.") + health_errors = my_health_check(dv.pressure) + if comm is not None: + health_errors = comm.allreduce(health_errors, op=MPI.LOR) + if health_errors: + if rank == 0: + logger.info("Fluid solution failed health check.") + raise HealthCheckError() + + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False + + if do_restart: + my_write_restart(step=step, t=t, state=state) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + my_write_viz(step=step, t=t, state=state, dv=dv) + + except MyError: + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + my_write_viz(step=step, t=t, state=state) + my_write_restart(step=step, t=t, state=state) + raise return state, dt @@ -300,14 +297,7 @@ def my_pre_step(step, t, dt, state): advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final, eos=eos, dim=dim) - - finish_tol = 1e-16 - if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(step=current_step, t=current_t, state=current_state, - do_viz=True, do_restart=True, - message="Simulation timestepping did not complete.") + state=current_state, t=current_t, t_final=t_final) # Dump the final data if rank == 0: @@ -321,6 +311,9 @@ def my_pre_step(step, t, dt, state): elif use_profiling: print(actx.tabulate_profiling_data()) + finish_tol = 1e-16 + assert (current_t - t_final) > finish_tol + if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 0980a528e..226094547 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -38,10 +38,7 @@ from mirgecom.euler import euler_operator -from mirgecom.simutil import ( - inviscid_sim_timestep, - generate_and_distribute_mesh -) +from mirgecom.simutil import generate_and_distribute_mesh from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -65,6 +62,18 @@ logger = logging.getLogger(__name__) +class MyError(Exception): + """Simple exception to kill the simulation.""" + + pass + + +class HealthCheckError(MyError): + """Simple exception to indicate a health check error.""" + + pass + + @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_leap=False, use_profiling=False, rst_step=None, rst_name=None, @@ -199,22 +208,6 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, if rank == 0: logger.info(init_message) - get_timestep = partial(inviscid_sim_timestep, discr=discr, - cfl=current_cfl, eos=eos, - t_final=t_final, constant_cfl=constant_cfl) - - def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, - message=None): - if rank == 0: - logger.info("Errors detected; attempting graceful exit.") - if do_viz: - my_write_viz(step=step, t=t, state=state) - if do_restart: - my_write_restart(step=step, t=t, state=state) - if message is None: - message = "Fatal simulation errors detected." - raise RuntimeError(message) - def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: dv = eos.dependent_vars(state) @@ -237,17 +230,18 @@ def my_write_restart(step, t, state): "state": state, "t": t, "step": step, + "order": order, "global_nelements": global_nelements, "num_parts": nparts } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) - def my_health_check(state, dv, exact): + def my_health_check(state, pressure, exact): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure, .9, 1.1): + if check_naninf_local(discr, "vol", pressure) \ + or check_range_local(discr, "vol", pressure, .9, 1.1): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") @@ -271,60 +265,63 @@ def my_post_step(step, t, dt, state): return state, dt def my_pre_step(step, t, dt, state): - dv = None - exact = None - pre_step_errors = False + try: + dv = None + exact = None - if logmgr: - logmgr.tick_before() - - from mirgecom.simutil import check_step - do_viz = check_step(step=step, interval=nviz) - do_restart = check_step(step=step, interval=nrestart) - do_health = check_step(step=step, interval=nhealth) - do_status = check_step(step=step, interval=nstatus) + if logmgr: + logmgr.tick_before() - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False + from mirgecom.simutil import check_step + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) + do_status = check_step(step=step, interval=nstatus) - if do_health: - dv = eos.dependent_vars(state) - exact = initializer(x_vec=nodes, eos=eos, t=t) - health_errors = my_health_check(state, dv, exact) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) - if health_errors and rank == 0: - logger.info("Fluid solution failed health check.") - pre_step_errors = pre_step_errors or health_errors - - if do_restart: - my_write_restart(step=step, t=t, state=state) - - if do_viz: - if dv is None: + if do_health: dv = eos.dependent_vars(state) - if exact is None: exact = initializer(x_vec=nodes, eos=eos, t=t) - resid = state - exact - my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, - resid=resid) - - if do_status: - if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) - from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact) - status_msg = ( - "------- errors=" - + ", ".join("%.3g" % en for en in component_errors)) + health_errors = my_health_check(state, dv.pressure, exact) + if comm is not None: + health_errors = comm.allreduce(health_errors, op=MPI.LOR) + if health_errors: + if rank == 0: + logger.info("Fluid solution failed health check.") + raise HealthCheckError() + + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False + + if do_restart: + my_write_restart(step=step, t=t, state=state) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + resid = state - exact + my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, + resid=resid) + + if do_status: + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + status_msg = ( + "------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + if rank == 0: + logger.info(status_msg) + + except MyError: if rank == 0: - logger.info(status_msg) - - if pre_step_errors: - my_graceful_exit(step=step, t=t, state=state, - do_viz=(not do_viz), do_restart=(not do_restart), - message="Error detected at prestep, exiting.") + logger.info("Errors detected; attempting graceful exit.") + my_write_viz(step=step, t=t, state=state) + my_write_restart(step=step, t=t, state=state) + raise return state, dt @@ -336,14 +333,7 @@ def my_rhs(t, state): advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, dt=current_dt, post_step_callback=my_post_step, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final, eos=eos, dim=dim) - - finish_tol = 1e-16 - if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(step=current_step, t=current_t, state=current_state, - do_viz=True, do_restart=True, - message="Simulation timestepping did not complete.") + state=current_state, t=current_t, t_final=t_final) # Dump the final data if rank == 0: @@ -361,6 +351,9 @@ def my_rhs(t, state): elif use_profiling: print(actx.tabulate_profiling_data()) + finish_tol = 1e-16 + assert (current_t - t_final) > finish_tol + if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index fc9782f0a..c572a4aee 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -24,7 +24,7 @@ THE SOFTWARE. """ import logging -import numpy as np +import numpy as np # noqa import pyopencl as cl import pyopencl.tools as cl_tools from functools import partial @@ -37,10 +37,7 @@ from mirgecom.euler import euler_operator -from mirgecom.simutil import ( - inviscid_sim_timestep, - generate_and_distribute_mesh -) +from mirgecom.simutil import generate_and_distribute_mesh from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -64,6 +61,18 @@ logger = logging.getLogger(__name__) +class MyError(Exception): + """Simple exception to kill the simulation.""" + + pass + + +class HealthCheckError(MyError): + """Simple exception to indicate a health check error.""" + + pass + + @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename="sod1d", @@ -184,18 +193,6 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, if rank == 0: logger.info(init_message) - def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, - message=None): - if rank == 0: - logger.info("Errors detected; attempting graceful exit.") - if do_viz: - my_write_viz(step=step, t=t, state=state) - if do_restart: - my_write_restart(step=step, t=t, state=state) - if message is None: - message = "Fatal simulation errors detected." - raise RuntimeError(message) - def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: dv = eos.dependent_vars(state) @@ -218,17 +215,18 @@ def my_write_restart(state, step, t): "state": state, "t": t, "step": step, + "order": order, "global_nelements": global_nelements, "num_parts": num_parts } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) - def my_health_check(state, dv, exact): + def my_health_check(state, pressure, exact): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure, .09, 1.1): + if check_naninf_local(discr, "vol", pressure) \ + or check_range_local(discr, "vol", pressure, .09, 1.1): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") @@ -252,67 +250,66 @@ def my_post_step(step, t, dt, state): return state, dt def my_pre_step(step, t, dt, state): - dv = None - exact = None - pre_step_errors = False + try: + dv = None + exact = None - if logmgr: - logmgr.tick_before() + if logmgr: + logmgr.tick_before() - from mirgecom.simutil import check_step - do_viz = check_step(step=step, interval=nviz) - do_restart = check_step(step=step, interval=nrestart) - do_health = check_step(step=step, interval=nhealth) - do_status = check_step(step=step, interval=nstatus) + from mirgecom.simutil import check_step + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) + do_status = check_step(step=step, interval=nstatus) - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False - - if do_health: - dv = eos.dependent_vars(state) - exact = initializer(x_vec=nodes, eos=eos, t=t) - health_errors = my_health_check(state, dv, exact) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) - if health_errors and rank == 0: - logger.info("Fluid solution failed health check.") - pre_step_errors = pre_step_errors or health_errors - - if do_restart: - my_write_restart(step=step, t=t, state=state) - - if do_viz: - if dv is None: + if do_health: dv = eos.dependent_vars(state) - if exact is None: exact = initializer(x_vec=nodes, eos=eos, t=t) - resid = state - exact - my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, - resid=resid) - - if do_status: - if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) - from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact) - status_msg = ( - "------- errors=" - + ", ".join("%.3g" % en for en in component_errors)) + health_errors = my_health_check(state, dv.pressure, exact) + if comm is not None: + health_errors = comm.allreduce(health_errors, op=MPI.LOR) + if health_errors: + if rank == 0: + logger.info("Fluid solution failed health check.") + raise HealthCheckError() + + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False + + if do_restart: + my_write_restart(step=step, t=t, state=state) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + resid = state - exact + my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, + resid=resid) + + if do_status: + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + status_msg = ( + "------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + if rank == 0: + logger.info(status_msg) + + except MyError: if rank == 0: - logger.info(status_msg) - - if pre_step_errors: - my_graceful_exit(step=step, t=t, state=state, - do_viz=(not do_viz), do_restart=(not do_restart), - message="Error detected at prestep, exiting.") + logger.info("Errors detected; attempting graceful exit.") + my_write_viz(step=step, t=t, state=state) + my_write_restart(step=step, t=t, state=state) + raise return state, dt - get_timestep = partial(inviscid_sim_timestep, discr=discr, - cfl=current_cfl, eos=eos, t_final=t_final, - constant_cfl=constant_cfl) - def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) @@ -321,14 +318,7 @@ def my_rhs(t, state): advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, dt=current_dt, post_step_callback=my_post_step, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final, eos=eos, dim=dim) - - finish_tol = 1e-16 - if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(step=current_step, t=current_t, state=current_state, - do_viz=True, do_restart=True, - message="Simulation timestepping did not complete.") + state=current_state, t=current_t, t_final=t_final) # Dump the final data if rank == 0: @@ -346,6 +336,9 @@ def my_rhs(t, state): elif use_profiling: print(actx.tabulate_profiling_data()) + finish_tol = 1e-16 + assert (current_t - t_final) > finish_tol + if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 06c1a94ae..8b89a908b 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -39,7 +39,6 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( - inviscid_sim_timestep, generate_and_distribute_mesh, check_step ) @@ -66,6 +65,18 @@ logger = logging.getLogger(__name__) +class MyError(Exception): + """Simple exception to kill the simulation.""" + + pass + + +class HealthCheckError(MyError): + """Simple exception to indicate a health check error.""" + + pass + + @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename="sod1d", @@ -205,18 +216,6 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, if rank == 0: logger.info(init_message) - def my_graceful_exit(step, t, state, do_viz=False, do_restart=False, - message=None): - if rank == 0: - logger.info("Errors detected; attempting graceful exit.") - if do_viz: - my_write_viz(step=step, t=t, state=state) - if do_restart: - my_write_restart(step=step, t=t, state=state) - if message is None: - message = "Fatal simulation errors detected." - raise RuntimeError(message) - def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: dv = eos.dependent_vars(state) @@ -239,17 +238,18 @@ def my_write_restart(step, t, state): "state": state, "t": t, "step": step, + "order": order, "global_nelements": global_nelements, "num_parts": num_parts } from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) - def my_health_check(state, dv, exact): + def my_health_check(state, pressure, exact): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure, .2, 1.02): + if check_naninf_local(discr, "vol", pressure) \ + or check_range_local(discr, "vol", pressure, .2, 1.02): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") @@ -273,66 +273,65 @@ def my_post_step(step, t, dt, state): return state, dt def my_pre_step(step, t, dt, state): - dv = None - exact = None - pre_step_errors = False - - if logmgr: - logmgr.tick_before() + try: + dv = None + exact = None - do_viz = check_step(step=step, interval=nviz) - do_restart = check_step(step=step, interval=nrestart) - do_health = check_step(step=step, interval=nhealth) - do_status = check_step(step=step, interval=nstatus) + if logmgr: + logmgr.tick_before() - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) + do_status = check_step(step=step, interval=nstatus) - if do_health: - dv = eos.dependent_vars(state) - exact = initializer(x_vec=nodes, eos=eos, t=t) - health_errors = my_health_check(state, dv, exact) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) - if health_errors and rank == 0: - logger.info("Fluid solution failed health check.") - pre_step_errors = pre_step_errors or health_errors - - if do_restart: - my_write_restart(step=step, t=t, state=state) - - if do_viz: - if dv is None: + if do_health: dv = eos.dependent_vars(state) - if exact is None: exact = initializer(x_vec=nodes, eos=eos, t=t) - resid = state - exact - my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, - resid=resid) - - if do_status: - if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) - from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact) - status_msg = ( - "------- errors=" - + ", ".join("%.3g" % en for en in component_errors)) + health_errors = my_health_check(state, dv.pressure, exact) + if comm is not None: + health_errors = comm.allreduce(health_errors, op=MPI.LOR) + if health_errors: + if rank == 0: + logger.info("Fluid solution failed health check.") + raise HealthCheckError() + + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False + + if do_restart: + my_write_restart(step=step, t=t, state=state) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + resid = state - exact + my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, + resid=resid) + + if do_status: + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + status_msg = ( + "------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + if rank == 0: + logger.info(status_msg) + + except MyError: if rank == 0: - logger.info(status_msg) - - if pre_step_errors: - my_graceful_exit(step=step, t=t, state=state, - do_viz=(not do_viz), do_restart=(not do_restart), - message="Error detected at prestep, exiting.") + logger.info("Errors detected; attempting graceful exit.") + my_write_viz(step=step, t=t, state=state) + my_write_restart(step=step, t=t, state=state) + raise return state, dt - get_timestep = partial(inviscid_sim_timestep, discr=discr, - cfl=current_cfl, eos=eos, t_final=t_final, - constant_cfl=constant_cfl) - def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) @@ -341,14 +340,7 @@ def my_rhs(t, state): advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, - get_timestep=get_timestep, state=current_state, - t=current_t, t_final=t_final, eos=eos, dim=dim) - - finish_tol = 1e-16 - if np.abs(current_t - t_final) > finish_tol: - my_graceful_exit(step=current_step, t=current_t, state=current_state, - do_viz=True, do_restart=True, - message="Simulation timestepping did not complete.") + state=current_state, t=current_t, t_final=t_final) # Dump the final data if rank == 0: @@ -366,6 +358,9 @@ def my_rhs(t, state): elif use_profiling: print(actx.tabulate_profiling_data()) + finish_tol = 1e-16 + assert (current_t - t_final) > finish_tol + if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) From 72505bc114d1079e387bc0aceafc149fb1874002 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 8 Jul 2021 22:09:27 -0500 Subject: [PATCH 237/385] Correct finished check, return a proper constant dt result, some other indentation errors, etc. --- examples/autoignition-mpi.py | 7 +++---- examples/lump-mpi.py | 5 +++-- examples/mixture-mpi.py | 5 +++-- examples/pulse-mpi.py | 5 +++-- examples/scalar-lump-mpi.py | 7 ++++--- examples/sod-mpi.py | 5 +++-- examples/vortex-mpi.py | 5 +++-- 7 files changed, 22 insertions(+), 17 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 71941774f..4c4cf14a1 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -382,8 +382,6 @@ def my_pre_step(step, t, dt, state): health_errors = my_health_check(dv) if comm is not None: health_errors = comm.allreduce(health_errors, op=MPI.LOR) - if step > 5: - health_errors = True if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") @@ -406,7 +404,8 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - return state, dt + t_remaining = max(0, t_final - t) + return state, min(dt, t_remaining) current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, @@ -427,7 +426,7 @@ def my_pre_step(step, t, dt, state): print(actx.tabulate_profiling_data()) finish_tol = 1e-16 - assert (current_t - t_final) > finish_tol + assert np.abs(current_t - t_final) < finish_tol if __name__ == "__main__": diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 7f5cd46cc..9bf8f133f 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -316,7 +316,8 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - return state, dt + t_remaining = max(0, t_final - t) + return state, min(dt, t_remaining) current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, @@ -341,7 +342,7 @@ def my_pre_step(step, t, dt, state): print(actx.tabulate_profiling_data()) finish_tol = 1e-16 - assert (current_t - t_final) > finish_tol + assert np.abs(current_t - t_final) < finish_tol if __name__ == "__main__": diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index ffc3005dc..8fc1f3c23 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -337,7 +337,8 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - return state, dt + t_remaining = max(0, t_final - t) + return state, min(dt, t_remaining) current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, @@ -362,7 +363,7 @@ def my_pre_step(step, t, dt, state): print(actx.tabulate_profiling_data()) finish_tol = 1e-16 - assert (current_t - t_final) > finish_tol + assert np.abs(current_t - t_final) < finish_tol if __name__ == "__main__": diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 8875979d8..7b72a2925 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -291,7 +291,8 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - return state, dt + t_remaining = max(0, t_final - t) + return state, min(dt, t_remaining) current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, @@ -312,7 +313,7 @@ def my_pre_step(step, t, dt, state): print(actx.tabulate_profiling_data()) finish_tol = 1e-16 - assert (current_t - t_final) > finish_tol + assert np.abs(current_t - t_final) < finish_tol if __name__ == "__main__": diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 226094547..b89afbb3b 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -287,7 +287,7 @@ def my_pre_step(step, t, dt, state): if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") - raise HealthCheckError() + raise HealthCheckError() if step == rst_step: # don't do viz or restart @ restart do_viz = False @@ -323,7 +323,8 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - return state, dt + t_remaining = max(0, t_final - t) + return state, min(dt, t_remaining) def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, @@ -352,7 +353,7 @@ def my_rhs(t, state): print(actx.tabulate_profiling_data()) finish_tol = 1e-16 - assert (current_t - t_final) > finish_tol + assert np.abs(current_t - t_final) < finish_tol if __name__ == "__main__": diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index c572a4aee..158c46be8 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -308,7 +308,8 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - return state, dt + t_remaining = max(0, t_final - t) + return state, min(dt, t_remaining) def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, @@ -337,7 +338,7 @@ def my_rhs(t, state): print(actx.tabulate_profiling_data()) finish_tol = 1e-16 - assert (current_t - t_final) > finish_tol + assert np.abs(current_t - t_final) < finish_tol if __name__ == "__main__": diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 8b89a908b..be1e6dd53 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -330,7 +330,8 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - return state, dt + t_remaining = max(0, t_final - t) + return state, min(dt, t_remaining) def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, @@ -359,7 +360,7 @@ def my_rhs(t, state): print(actx.tabulate_profiling_data()) finish_tol = 1e-16 - assert (current_t - t_final) > finish_tol + assert np.abs(current_t - t_final) < finish_tol if __name__ == "__main__": From 3c3bbd521d96cb21bde08b9f1062f6192e3da1f4 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 9 Jul 2021 08:39:13 -0500 Subject: [PATCH 238/385] Correct and enhance the restart processing to demonstrate change-of-order on restart. --- examples/autoignition-mpi.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 4c4cf14a1..3450ea78b 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -84,7 +84,7 @@ class HealthCheckError(MyError): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=False, use_leap=False, use_profiling=False, casename="autoignition", - rst_step=None, rst_name=None): + rst_filename=None): """Drive example.""" cl_ctx = ctx_factory() @@ -150,8 +150,9 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) - if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) + restarting = rst_filename is not None + if restarting: # read the grid from restart data + rst_fname = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_fname) @@ -159,6 +160,9 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["num_parts"] == nproc + rst_time = restart_data["t"] + rst_step = restart_data["step"] + rst_order = restart_data["order"] else: # generate the grid from scratch from meshmode.mesh.generation import generate_regular_rect_mesh box_ll = -0.005 @@ -264,13 +268,22 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, my_boundary = AdiabaticSlipBoundary() boundaries = {BTAG_ALL: my_boundary} - if rst_step: - current_t = restart_data["t"] + if restarting: current_step = rst_step - current_state = restart_data["state"] + current_t = rst_time if logmgr: from mirgecom.logging_quantities import logmgr_set_time logmgr_set_time(logmgr, current_step, current_t) + if order == rst_order: + current_state = restart_data["state"] + else: + rst_state = restart_data["state"] + old_discr = EagerDGDiscretization(actx, local_mesh, order=rst_order, + mpi_communicator=comm) + from meshmode.discretization.connection import make_same_mesh_connection + connection = make_same_mesh_connection(actx, discr.discr_from_dd("vol"), + old_discr.discr_from_dd("vol")) + current_state = connection(rst_state) else: # Set the current state from time 0 current_state = initializer(eos=eos, x_vec=nodes, t=0) From 41f78ba8d1366ade0b14dc3a17ad5cd6ee0d4bf1 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 9 Jul 2021 11:23:09 -0500 Subject: [PATCH 239/385] Update restart logic, add missing final dump advice. --- examples/autoignition-mpi.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 3450ea78b..576146edd 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -334,17 +334,21 @@ def my_write_viz(step, t, state, dv=None, production_rates=None): def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) - rst_data = { - "local_mesh": local_mesh, - "state": state, - "t": t, - "step": step, - "order": order, - "global_nelements": global_nelements, - "num_parts": nproc - } - from mirgecom.restart import write_restart_file - write_restart_file(actx, rst_data, rst_fname, comm) + if rst_fname == rst_filename: + if rank == 0: + logger.info("Skipping overwrite of restart file.") + else: + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": nproc + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(dv): health_error = False @@ -386,10 +390,6 @@ def my_pre_step(step, t, dt, state): do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False - if do_health: dv = eos.dependent_vars(state) health_errors = my_health_check(dv) @@ -427,6 +427,9 @@ def my_pre_step(step, t, dt, state): state=current_state, t=current_t, t_final=t_final) # Dump the final data + if rank == 0: + logger.info("Checkpointing final state ...") + final_dv = eos.dependent_vars(current_state) final_dm = eos.get_production_rates(current_state) my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, From 5c3ab55e04abaaa5a412836bf5e1f06935edf6d0 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 9 Jul 2021 11:28:05 -0500 Subject: [PATCH 240/385] Use built-in exceptions instead of custom ones. --- examples/autoignition-mpi.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 576146edd..01fd0453f 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -69,18 +69,6 @@ logger = logging.getLogger(__name__) -class MyError(Exception): - """Simple exception to kill the simulation.""" - - pass - - -class HealthCheckError(MyError): - """Simple exception to indicate a health check error.""" - - pass - - @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=False, use_leap=False, use_profiling=False, casename="autoignition", @@ -395,10 +383,12 @@ def my_pre_step(step, t, dt, state): health_errors = my_health_check(dv) if comm is not None: health_errors = comm.allreduce(health_errors, op=MPI.LOR) + if step == 5: # quick test + health_errors = True if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") - raise HealthCheckError() + raise if do_restart: my_write_restart(step=step, t=t, state=state) @@ -410,12 +400,12 @@ def my_pre_step(step, t, dt, state): my_write_viz(step=step, t=t, state=state, dv=dv, production_rates=production_rates) - except MyError: + except BaseException: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, state=state) my_write_restart(step=step, t=t, state=state) - raise + raise RuntimeError t_remaining = max(0, t_final - t) return state, min(dt, t_remaining) From c544dddac735625d205b6bc8b82490fbf0c68400 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 9 Jul 2021 11:35:08 -0500 Subject: [PATCH 241/385] Satisfy pylint to raise an actual named exception inside try. --- examples/autoignition-mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 01fd0453f..2b1c2740d 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -388,7 +388,7 @@ def my_pre_step(step, t, dt, state): if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") - raise + raise Exception if do_restart: my_write_restart(step=step, t=t, state=state) From 810ab46f8be29be096c5f246ffda9d6a7a29155a Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 9 Jul 2021 11:56:28 -0500 Subject: [PATCH 242/385] Fix up the documentation to have the correct signature for callbacks Co-authored-by: Matt Smith --- mirgecom/steppers.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 7a60f5600..f4913e352 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -67,11 +67,11 @@ def _advance_state_stepper_func(rhs, timestepper, Step number from which to start pre_step_callback An optional user-defined function, with signature: - ``state, dt = pre_step_callback(state, step, t, dt)``, + ``state, dt = pre_step_callback(step, t, dt, state)``, to be called before the timestepper is called for that particular step. post_step_callback An optional user-defined function, with signature: - ``state, dt = post_step_callback(state, step, t, dt)``, + ``state, dt = post_step_callback(step, t, dt, state)``, to be called after the timestepper is called for that particular step. Returns @@ -148,11 +148,11 @@ def _advance_state_leap(rhs, timestepper, state, Step number from which to start pre_step_callback An optional user-defined function, with signature: - ``state, dt = pre_step_callback(state, step, t, dt)``, + ``state, dt = pre_step_callback(step, t, dt, state)``, to be called before the timestepper is called for that particular step. post_step_callback An optional user-defined function, with signature: - ``state, dt = post_step_callback(state, step, t, dt)``, + ``state, dt = post_step_callback(step, t, dt, state)``, to be called after the timestepper is called for that particular step. Returns @@ -282,11 +282,11 @@ def advance_state(rhs, timestepper, state, t_final, Step number from which to start pre_step_callback An optional user-defined function, with signature: - ``state, dt = pre_step_callback(state, step, t, dt)``, + ``state, dt = pre_step_callback(step, t, dt, state)``, to be called before the timestepper is called for that particular step. post_step_callback An optional user-defined function, with signature: - ``state, dt = post_step_callback(state, step, t, dt)``, + ``state, dt = post_step_callback(step, t, dt, state)``, to be called after the timestepper is called for that particular step. Returns From 549a8dd6732ce0098f53d90a9b8aae01132a7549 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 9 Jul 2021 11:58:13 -0500 Subject: [PATCH 243/385] Massage exceptions a bit so that the error messages are a little more informative, cleaner. --- examples/autoignition-mpi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 2b1c2740d..a9e44c47e 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -388,7 +388,7 @@ def my_pre_step(step, t, dt, state): if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") - raise Exception + raise RuntimeError("Failed health check.") if do_restart: my_write_restart(step=step, t=t, state=state) @@ -405,7 +405,7 @@ def my_pre_step(step, t, dt, state): logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, state=state) my_write_restart(step=step, t=t, state=state) - raise RuntimeError + raise t_remaining = max(0, t_final - t) return state, min(dt, t_remaining) From af009c33c3bffd6008c00a6234e73923dc926666 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 9 Jul 2021 12:10:41 -0500 Subject: [PATCH 244/385] Rearrange function def order for consistency and less "tism" activation. --- examples/autoignition-mpi.py | 28 ++++++++++++++-------------- examples/lump-mpi.py | 26 +++++++++++++------------- examples/mixture-mpi.py | 26 +++++++++++++------------- examples/pulse-mpi.py | 26 +++++++++++++------------- examples/scalar-lump-mpi.py | 18 +++++++++--------- examples/sod-mpi.py | 18 +++++++++--------- examples/vortex-mpi.py | 18 +++++++++--------- 7 files changed, 80 insertions(+), 80 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index a9e44c47e..5f684831b 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -352,20 +352,6 @@ def my_health_check(dv): return health_error - def my_rhs(t, state): - return (euler_operator(discr, cv=state, t=t, - boundaries=boundaries, eos=eos) - + eos.get_species_source_terms(state)) - - def my_post_step(step, t, dt, state): - # Logmgr needs to know about EOS, dt, dim? - # imo this is a design/scope flaw - if logmgr: - set_dt(logmgr, dt) - set_sim_state(logmgr, dim, state, eos) - logmgr.tick_after() - return state, dt - def my_pre_step(step, t, dt, state): try: dv = None @@ -410,6 +396,20 @@ def my_pre_step(step, t, dt, state): t_remaining = max(0, t_final - t) return state, min(dt, t_remaining) + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + + def my_rhs(t, state): + return (euler_operator(discr, cv=state, t=t, + boundaries=boundaries, eos=eos) + + eos.get_species_source_terms(state)) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 9bf8f133f..7deb72768 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -244,19 +244,6 @@ def my_health_check(dv, state, exact): return health_error - def my_rhs(t, state): - return euler_operator(discr, cv=state, t=t, - boundaries=boundaries, eos=eos) - - def my_post_step(step, t, dt, state): - # Logmgr needs to know about EOS, dt, dim? - # imo this is a design/scope flaw - if logmgr: - set_dt(logmgr, dt) - set_sim_state(logmgr, dim, state, eos) - logmgr.tick_after() - return state, dt - def my_pre_step(step, t, dt, state): try: dv = None @@ -319,6 +306,19 @@ def my_pre_step(step, t, dt, state): t_remaining = max(0, t_final - t) return state, min(dt, t_remaining) + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + + def my_rhs(t, state): + return euler_operator(discr, cv=state, t=t, + boundaries=boundaries, eos=eos) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 8fc1f3c23..ddfa69bf1 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -265,19 +265,6 @@ def my_health_check(state, dv, exact): return health_error - def my_rhs(t, state): - return euler_operator(discr, cv=state, t=t, - boundaries=boundaries, eos=eos) - - def my_post_step(step, t, dt, state): - # Logmgr needs to know about EOS, dt, dim? - # imo this is a design/scope flaw - if logmgr: - set_dt(logmgr, dt) - set_sim_state(logmgr, dim, state, eos) - logmgr.tick_after() - return state, dt - def my_pre_step(step, t, dt, state): try: dv = None @@ -340,6 +327,19 @@ def my_pre_step(step, t, dt, state): t_remaining = max(0, t_final - t) return state, min(dt, t_remaining) + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + + def my_rhs(t, state): + return euler_operator(discr, cv=state, t=t, + boundaries=boundaries, eos=eos) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 7b72a2925..39a407c87 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -237,19 +237,6 @@ def my_health_check(pressure): logger.info(f"{rank=}: Invalid pressure data found.") return health_error - def my_rhs(t, state): - return euler_operator(discr, cv=state, t=t, - boundaries=boundaries, eos=eos) - - def my_post_step(step, t, dt, state): - # Logmgr needs to know about EOS, dt, dim? - # imo this is a design/scope flaw - if logmgr: - set_dt(logmgr, dt) - set_sim_state(logmgr, dim, state, eos) - logmgr.tick_after() - return state, dt - def my_pre_step(step, t, dt, state): try: dv = None @@ -294,6 +281,19 @@ def my_pre_step(step, t, dt, state): t_remaining = max(0, t_final - t) return state, min(dt, t_remaining) + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + + def my_rhs(t, state): + return euler_operator(discr, cv=state, t=t, + boundaries=boundaries, eos=eos) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index b89afbb3b..e3d44f7a7 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -255,15 +255,6 @@ def my_health_check(state, pressure, exact): return health_error - def my_post_step(step, t, dt, state): - # Logmgr needs to know about EOS, dt, dim? - # imo this is a design/scope flaw - if logmgr: - set_dt(logmgr, dt) - set_sim_state(logmgr, dim, state, eos) - logmgr.tick_after() - return state, dt - def my_pre_step(step, t, dt, state): try: dv = None @@ -326,6 +317,15 @@ def my_pre_step(step, t, dt, state): t_remaining = max(0, t_final - t) return state, min(dt, t_remaining) + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 158c46be8..07ada666c 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -240,15 +240,6 @@ def my_health_check(state, pressure, exact): return health_error - def my_post_step(step, t, dt, state): - # Logmgr needs to know about EOS, dt, dim? - # imo this is a design/scope flaw - if logmgr: - set_dt(logmgr, dt) - set_sim_state(logmgr, dim, state, eos) - logmgr.tick_after() - return state, dt - def my_pre_step(step, t, dt, state): try: dv = None @@ -311,6 +302,15 @@ def my_pre_step(step, t, dt, state): t_remaining = max(0, t_final - t) return state, min(dt, t_remaining) + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index be1e6dd53..ede65fc3e 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -263,15 +263,6 @@ def my_health_check(state, pressure, exact): return health_error - def my_post_step(step, t, dt, state): - # Logmgr needs to know about EOS, dt, dim? - # imo this is a design/scope flaw - if logmgr: - set_dt(logmgr, dt) - set_sim_state(logmgr, dim, state, eos) - logmgr.tick_after() - return state, dt - def my_pre_step(step, t, dt, state): try: dv = None @@ -333,6 +324,15 @@ def my_pre_step(step, t, dt, state): t_remaining = max(0, t_final - t) return state, min(dt, t_remaining) + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) From 5306e301e92c3764c390e4e6fe29cec1080a4272 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 9 Jul 2021 13:53:17 -0500 Subject: [PATCH 245/385] Tweak exception handling to our liking. --- examples/autoignition-mpi.py | 12 ++++++++---- examples/lump-mpi.py | 12 +++--------- examples/mixture-mpi.py | 12 +++--------- examples/pulse-mpi.py | 12 +++--------- examples/scalar-lump-mpi.py | 12 +++--------- examples/sod-mpi.py | 12 +++--------- examples/vortex-mpi.py | 12 +++--------- 7 files changed, 26 insertions(+), 58 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 5f684831b..9e200d934 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -69,6 +69,12 @@ logger = logging.getLogger(__name__) +class MyRuntimeError(RuntimeError): + """Simple exception for fatal driver errors.""" + + pass + + @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=False, use_leap=False, use_profiling=False, casename="autoignition", @@ -369,12 +375,10 @@ def my_pre_step(step, t, dt, state): health_errors = my_health_check(dv) if comm is not None: health_errors = comm.allreduce(health_errors, op=MPI.LOR) - if step == 5: # quick test - health_errors = True if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") - raise RuntimeError("Failed health check.") + raise MyRuntimeError("Failed simulation health check.") if do_restart: my_write_restart(step=step, t=t, state=state) @@ -386,7 +390,7 @@ def my_pre_step(step, t, dt, state): my_write_viz(step=step, t=t, state=state, dv=dv, production_rates=production_rates) - except BaseException: + except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, state=state) diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 7deb72768..6f65263c9 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -61,18 +61,12 @@ logger = logging.getLogger(__name__) -class MyError(Exception): +class MyRuntimeError(RuntimeError): """Simple exception to kill the simulation.""" pass -class HealthCheckError(MyError): - """Simple exception to indicate a health check error.""" - - pass - - @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_leap=False, use_profiling=False, rst_step=None, rst_name=None, @@ -267,7 +261,7 @@ def my_pre_step(step, t, dt, state): if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") - raise HealthCheckError() + raise MyRuntimeError("Failed solution health check.") if step == rst_step: # don't do viz or restart @ restart do_viz = False @@ -296,7 +290,7 @@ def my_pre_step(step, t, dt, state): if rank == 0: logger.info(status_msg) - except MyError: + except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, state=state) diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index ddfa69bf1..381d95dd0 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -64,18 +64,12 @@ logger = logging.getLogger(__name__) -class MyError(Exception): +class MyRuntimeError(RuntimeError): """Simple exception to kill the simulation.""" pass -class HealthCheckError(MyError): - """Simple exception to indicate a health check error.""" - - pass - - @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_leap=False, use_profiling=False, rst_step=None, rst_name=None, @@ -288,7 +282,7 @@ def my_pre_step(step, t, dt, state): if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") - raise HealthCheckError() + raise MyRuntimeError("Failed simulation health check.") if step == rst_step: # don't do viz or restart @ restart do_viz = False @@ -317,7 +311,7 @@ def my_pre_step(step, t, dt, state): if rank == 0: logger.info(status_msg) - except MyError: + except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, state=state) diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 39a407c87..984b64fed 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -67,18 +67,12 @@ logger = logging.getLogger(__name__) -class MyError(Exception): +class MyRuntimeError(RuntimeError): """Simple exception to kill the simulation.""" pass -class HealthCheckError(MyError): - """Simple exception to indicate a health check error.""" - - pass - - @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename="pulse", @@ -257,7 +251,7 @@ def my_pre_step(step, t, dt, state): if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") - raise HealthCheckError() + raise MyRuntimeError("Failed simulation health check.") if step == rst_step: # don't do viz or restart @ restart do_viz = False @@ -271,7 +265,7 @@ def my_pre_step(step, t, dt, state): dv = eos.dependent_vars(state) my_write_viz(step=step, t=t, state=state, dv=dv) - except MyError: + except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, state=state) diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index e3d44f7a7..a5d34a44c 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -62,18 +62,12 @@ logger = logging.getLogger(__name__) -class MyError(Exception): +class MyRuntimeError(RuntimeError): """Simple exception to kill the simulation.""" pass -class HealthCheckError(MyError): - """Simple exception to indicate a health check error.""" - - pass - - @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_leap=False, use_profiling=False, rst_step=None, rst_name=None, @@ -278,7 +272,7 @@ def my_pre_step(step, t, dt, state): if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") - raise HealthCheckError() + raise MyRuntimeError("Failed simulation health check.") if step == rst_step: # don't do viz or restart @ restart do_viz = False @@ -307,7 +301,7 @@ def my_pre_step(step, t, dt, state): if rank == 0: logger.info(status_msg) - except MyError: + except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, state=state) diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 07ada666c..21be15bdf 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -61,18 +61,12 @@ logger = logging.getLogger(__name__) -class MyError(Exception): +class MyRuntimeError(RuntimeError): """Simple exception to kill the simulation.""" pass -class HealthCheckError(MyError): - """Simple exception to indicate a health check error.""" - - pass - - @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename="sod1d", @@ -263,7 +257,7 @@ def my_pre_step(step, t, dt, state): if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") - raise HealthCheckError() + raise MyRuntimeError("Failed simulation health check.") if step == rst_step: # don't do viz or restart @ restart do_viz = False @@ -292,7 +286,7 @@ def my_pre_step(step, t, dt, state): if rank == 0: logger.info(status_msg) - except MyError: + except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, state=state) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index ede65fc3e..383bedb7b 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -65,18 +65,12 @@ logger = logging.getLogger(__name__) -class MyError(Exception): +class MyRuntimeError(RuntimeError): """Simple exception to kill the simulation.""" pass -class HealthCheckError(MyError): - """Simple exception to indicate a health check error.""" - - pass - - @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename="sod1d", @@ -285,7 +279,7 @@ def my_pre_step(step, t, dt, state): if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") - raise HealthCheckError() + raise MyRuntimeError("Failed simulation health check.") if step == rst_step: # don't do viz or restart @ restart do_viz = False @@ -314,7 +308,7 @@ def my_pre_step(step, t, dt, state): if rank == 0: logger.info(status_msg) - except MyError: + except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") my_write_viz(step=step, t=t, state=state) From 5bd30d8d511c48d5155e0fd58fca93bffe5becc8 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 9 Jul 2021 14:25:14 -0500 Subject: [PATCH 246/385] Add syncing utility and use it in examples. --- examples/autoignition-mpi.py | 7 +++---- examples/lump-mpi.py | 8 +++++--- examples/mixture-mpi.py | 6 +++--- examples/pulse-mpi.py | 6 +++--- examples/scalar-lump-mpi.py | 6 +++--- examples/sod-mpi.py | 6 +++--- examples/vortex-mpi.py | 6 +++--- mirgecom/simutil.py | 11 +++++++++++ 8 files changed, 34 insertions(+), 22 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 9e200d934..893b24e63 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -352,7 +352,7 @@ def my_health_check(dv): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") - if check_range_local(discr, "vol", dv.temperature, 1.4e3, 3.3e3): + if check_range_local(discr, "vol", dv.temperature, 1.498e3, 1.52e3): health_error = True logger.info(f"{rank=}: Invalid temperature data found.") @@ -372,9 +372,8 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - health_errors = my_health_check(dv) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) + from mirgecom.simutil import allsync + health_errors = allsync(my_health_check(dv), comm, op=MPI.LOR) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 6f65263c9..75eae69c4 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -255,9 +255,11 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) exact = initializer(x_vec=nodes, eos=eos, t=t) - health_errors = my_health_check(dv=dv, state=state, exact=exact) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) + from mirgecom.simutil import allsync + health_errors = allsync( + my_health_check(dv=dv, state=state, exact=exact), + comm, op=MPI.LOR + ) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 381d95dd0..eb1ec13ea 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -276,9 +276,9 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) exact = initializer(x_vec=nodes, eos=eos, t=t) - health_errors = my_health_check(state, dv, exact) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) + from mirgecom.simutil import allsync + health_errors = allsync(my_health_check(state, dv, exact), comm, + op=MPI.LOR) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 984b64fed..7afdda6d3 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -245,9 +245,9 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - health_errors = my_health_check(dv.pressure) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) + from mirgecom.simutil import allsync + health_errors = allsync(my_health_check(dv.pressure), comm, + op=MPI.LOR) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index a5d34a44c..c7ce42fdc 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -266,9 +266,9 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) exact = initializer(x_vec=nodes, eos=eos, t=t) - health_errors = my_health_check(state, dv.pressure, exact) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) + from mirgecom.simutil import allsync + health_errors = allsync(my_health_check(state, dv.pressure, exact), + comm, op=MPI.LOR) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 21be15bdf..9e3165f4e 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -251,9 +251,9 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) exact = initializer(x_vec=nodes, eos=eos, t=t) - health_errors = my_health_check(state, dv.pressure, exact) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) + from mirgecom.simutil import allsync + health_errors = allsync(my_health_check(state, dv.pressure, exact), + comm, op=MPI.LOR) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 383bedb7b..17ab5a830 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -273,9 +273,9 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) exact = initializer(x_vec=nodes, eos=eos, t=t) - health_errors = my_health_check(state, dv.pressure, exact) - if comm is not None: - health_errors = comm.allreduce(health_errors, op=MPI.LOR) + from mirgecom.simutil import allsync + health_errors = allsync(my_health_check(state, dv.pressure, exact), + comm, op=MPI.LOR) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 50bac0922..ac6219fa7 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -6,6 +6,7 @@ .. autofunction:: check_step .. autofunction:: inviscid_sim_timestep .. autofunction:: write_visfile +.. autofunction:: allsync Diagnostic utilities -------------------- @@ -122,6 +123,16 @@ def write_visfile(discr, io_fields, visualizer, vizname, ) +def allsync(local_values, comm=None, op=None): + """Perform allreduce if MPI comm is provided.""" + if comm is None: + return local_values + if op is None: + from mpi4py import MPI + op = MPI.MAX + return comm.allreduce(local_values, op=op) + + def check_range_local(discr, dd, field, min_value, max_value): """Check for any negative values.""" return ( From 56df2d87cdc7f1e0196a7d9bc7d86e0a46b3b501 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 9 Jul 2021 16:39:32 -0500 Subject: [PATCH 247/385] Notify of which examples failed. --- examples/run_examples.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/run_examples.sh b/examples/run_examples.sh index c6c195a99..71a57d69d 100755 --- a/examples/run_examples.sh +++ b/examples/run_examples.sh @@ -7,6 +7,7 @@ origin=$(pwd) examples_dir=${1-$origin} declare -i exitcode=0 echo "Running examples in $examples_dir ..." +failed_examples="" for example in $examples_dir/*.py do if [[ "$example" == *"-mpi.py" ]] @@ -23,6 +24,7 @@ do else ((exitcode=exitcode+1)) echo "Example $example failed." + failed_examples="$failed_examples $example" fi done echo "Done running examples!" @@ -30,7 +32,7 @@ if [[ $exitcode -eq 0 ]] then echo "No errors." else - echo "Errors detected ($exitcode)." + echo "Errors detected ($exitcode):($failed_examples )" exit $exitcode fi #rm -f examples/*.vtu From 53a220d3a0dab9751ccaa774abddb0c4b58424ad Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 9 Jul 2021 19:04:50 -0500 Subject: [PATCH 248/385] Fix merge error --- mirgecom/simutil.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 4467d0cfe..6a52d80c7 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -123,37 +123,6 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, ) return min(t_remaining, mydt) - if vis_timer: - ctm = vis_timer.start_sub_timer() - else: - ctm = nullcontext() - - with ctm: - visualizer.write_parallel_vtk_file( - comm, rank_fn, io_fields, - overwrite=overwrite, - par_manifest_filename=make_par_fname( - basename=vizname, step=step, t=t - ) - ) - - -def allsync(local_values, comm=None, op=None): - """Perform allreduce if MPI comm is provided.""" - if comm is None: - return local_values - if op is None: - from mpi4py import MPI - op = MPI.MAX - return comm.allreduce(local_values, op=op) - - -def check_range_local(discr, dd, field, min_value, max_value): - """Check for any negative values.""" - return ( - op.nodal_min_loc(discr, dd, field) < min_value - or op.nodal_max_loc(discr, dd, field) > max_value - ) def write_visfile(discr, io_fields, visualizer, vizname, step=0, t=0, overwrite=False, vis_timer=None): From f781d80ef0801fb4e41419c99e3652443b7db9ca Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 9 Jul 2021 23:28:57 -0500 Subject: [PATCH 249/385] Clean up a couple of drivers, move toward constant cfl support --- examples/autoignition-mpi.py | 29 ++++++++++++++++++++------ examples/vortex-mpi.py | 40 +++++++++++++++++++++--------------- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 40a0904c9..5fdab4db0 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -314,7 +314,16 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, f" {eq_pressure=}, {eq_temperature=}," f" {eq_density=}, {eq_mass_fractions=}") - def my_write_viz(step, t, state, ts_field, dv=None, production_rates=None): + def my_write_status(dt, cfl): + if constant_cfl: + status_msg = f"------ {dt=}" + else: + status_msg = f"------ {cfl=}" + if rank == 0: + logger.info(status_msg) + + def my_write_viz(step, t, dt, state, ts_field=None, dv=None, + production_rates=None): if dv is None: dv = eos.dependent_vars(state) if production_rates is None: @@ -322,6 +331,8 @@ def my_write_viz(step, t, state, ts_field, dv=None, production_rates=None): viz_fields = [("cv", state), ("dv", dv), ("production_rates", production_rates)] + if ts_field is None: + ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=state) if constant_cfl: viz_fields.append(("local_dt", ts_field)) else: @@ -388,6 +399,7 @@ def my_pre_step(step, t, dt, state): do_viz = check_step(step=step, interval=nviz) do_restart = check_step(step=step, interval=nrestart) do_health = check_step(step=step, interval=nhealth) + do_status = check_step(step=step, interval=nstatus) if do_health: dv = eos.dependent_vars(state) @@ -400,6 +412,9 @@ def my_pre_step(step, t, dt, state): ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=state) + if do_status: + my_write_status(dt, cfl) + if do_restart: my_write_restart(step=step, t=t, state=state) @@ -407,7 +422,7 @@ def my_pre_step(step, t, dt, state): production_rates = eos.get_production_rates(state) if dv is None: dv = eos.dependent_vars(state) - my_write_viz(step=step, t=t, state=state, dv=dv, + my_write_viz(step=step, t=t, dt=dt, state=state, dv=dv, production_rates=production_rates, ts_field=ts_field) @@ -418,8 +433,7 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - t_remaining = max(0, t_final - t) - return state, min(dt, t_remaining) + return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? @@ -447,8 +461,11 @@ def my_rhs(t, state): final_dv = eos.dependent_vars(current_state) final_dm = eos.get_production_rates(current_state) - my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, - production_rates=final_dm) + ts_field, cfl, dt = my_get_timestep(t=current_t, dt=current_dt, + state=current_state) + my_write_viz(step=current_step, t=current_t, dt=dt, state=current_state, + dv=final_dv, production_rates=final_dm, ts_field=ts_field) + my_write_status(dt=dt, cfl=cfl) my_write_restart(step=current_step, t=current_t, state=current_state) if logmgr: diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 17ab5a830..4831d0c0c 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -210,6 +210,16 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, if rank == 0: logger.info(init_message) + def my_write_status(component_errors, cfl=None): + status_msg = "" + if cfl is not None: + status_msg = f"------ {cfl=}\n" + status_msg += ( + "------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + if rank == 0: + logger.info(status_msg) + def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: dv = eos.dependent_vars(state) @@ -239,7 +249,7 @@ def my_write_restart(step, t, state): from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) - def my_health_check(state, pressure, exact): + def my_health_check(state, pressure, exact, component_errors): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", pressure) \ @@ -247,8 +257,6 @@ def my_health_check(state, pressure, exact): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") - from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact) exittol = .1 if max(component_errors) > exittol: health_error = True @@ -273,9 +281,13 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) from mirgecom.simutil import allsync - health_errors = allsync(my_health_check(state, dv.pressure, exact), - comm, op=MPI.LOR) + health_errors = allsync( + my_health_check(state, dv.pressure, exact, component_errors), + comm, op=MPI.LOR + ) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") @@ -288,6 +300,13 @@ def my_pre_step(step, t, dt, state): if do_restart: my_write_restart(step=step, t=t, state=state) + if do_status: + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + my_write_status(component_errors) + if do_viz: if dv is None: dv = eos.dependent_vars(state) @@ -297,17 +316,6 @@ def my_pre_step(step, t, dt, state): my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, resid=resid) - if do_status: - if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) - from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact) - status_msg = ( - "------- errors=" - + ", ".join("%.3g" % en for en in component_errors)) - if rank == 0: - logger.info(status_msg) - except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") From b8114cde68bb3912dc9239e31edc3e7302230043 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 10 Jul 2021 07:54:10 -0500 Subject: [PATCH 250/385] add missing arg to write_vis --- examples/autoignition-mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 5fdab4db0..4848f1de1 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -429,7 +429,7 @@ def my_pre_step(step, t, dt, state): except MyRuntimeError: if rank == 0: logger.info("Errors detected; attempting graceful exit.") - my_write_viz(step=step, t=t, state=state) + my_write_viz(step=step, t=t, dt=dt, state=state) my_write_restart(step=step, t=t, state=state) raise From c523fa454d5ee4edb349c78742327f2614ff528d Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 10 Jul 2021 07:55:18 -0500 Subject: [PATCH 251/385] Remove superfluous constant CFL example. --- examples/mixture-fixed-cfl-mpi.py | 315 ------------------------------ 1 file changed, 315 deletions(-) delete mode 100644 examples/mixture-fixed-cfl-mpi.py diff --git a/examples/mixture-fixed-cfl-mpi.py b/examples/mixture-fixed-cfl-mpi.py deleted file mode 100644 index a7200d2aa..000000000 --- a/examples/mixture-fixed-cfl-mpi.py +++ /dev/null @@ -1,315 +0,0 @@ -"""Demonstrate combustive mixture with constant CFL mode.""" - -__copyright__ = """ -Copyright (C) 2020 University of Illinois Board of Trustees -""" - -__license__ = """ -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -""" -import logging -import numpy as np -import pyopencl as cl -import pyopencl.tools as cl_tools -from functools import partial - -from meshmode.array_context import PyOpenCLArrayContext -from meshmode.dof_array import thaw -from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa -from grudge.eager import EagerDGDiscretization -from grudge.shortcuts import make_visualizer - - -from mirgecom.euler import euler_operator -from mirgecom.simutil import ( - check_step, - generate_and_distribute_mesh, - ExactSolutionMismatch -) -from mirgecom.inviscid import ( - get_inviscid_timestep, - get_inviscid_cfl -) -from mirgecom.io import make_init_message -from mirgecom.mpi import mpi_entry_point -from mirgecom.integrators import rk4_step -from mirgecom.steppers import advance_state -from mirgecom.boundary import AdiabaticSlipBoundary -from mirgecom.initializers import MixtureInitializer -from mirgecom.eos import PyrometheusMixture - -import cantera -import pyrometheus as pyro - -logger = logging.getLogger(__name__) - - -@mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False): - """Drive example.""" - cl_ctx = ctx_factory() - queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - - dim = 2 - nel_1d = 8 - order = 1 - - # This example runs only 3 steps by default (to keep CI ~short) - # With the mixture defined below, equilibrium is achieved at ~40ms - # To run to equlibrium, set t_final >= 40ms. - t_final = 1e-7 - current_cfl = 0.01 - velocity = np.zeros(shape=(dim,)) - current_dt = 1e-9 - current_t = 0 - constant_cfl = True - nstatus = 1 - nviz = 2 - rank = 0 - checkpoint_t = current_t - current_step = 0 - if use_leap: - from leap.rk import RK4MethodBuilder - timestepper = RK4MethodBuilder("state") - else: - timestepper = rk4_step - box_ll = -0.005 - box_ur = 0.005 - error_state = False - debug = False - - from mpi4py import MPI - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - - from meshmode.mesh.generation import generate_regular_rect_mesh - generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,) * dim, - b=(box_ur,) * dim, nelements_per_axis=(nel_1d,) * dim) - local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) - local_nelements = local_mesh.nelements - - discr = EagerDGDiscretization( - actx, local_mesh, order=order, mpi_communicator=comm - ) - nodes = thaw(actx, discr.nodes()) - - # {{{ Set up initial state using Cantera - - # Use Cantera for initialization - # -- Pick up a CTI for the thermochemistry config - # --- Note: Users may add their own CTI file by dropping it into - # --- mirgecom/mechanisms alongside the other CTI files. - from mirgecom.mechanisms import get_mechanism_cti - mech_cti = get_mechanism_cti("uiuc") - - cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti) - nspecies = cantera_soln.n_species - - # Initial temperature, pressure, and mixutre mole fractions are needed to - # set up the initial state in Cantera. - init_temperature = 1500.0 # Initial temperature hot enough to burn - # Parameters for calculating the amounts of fuel, oxidizer, and inert species - equiv_ratio = 1.0 - ox_di_ratio = 0.21 - stoich_ratio = 3.0 - # Grab the array indices for the specific species, ethylene, oxygen, and nitrogen - i_fu = cantera_soln.species_index("C2H4") - i_ox = cantera_soln.species_index("O2") - i_di = cantera_soln.species_index("N2") - x = np.zeros(nspecies) - # Set the species mole fractions according to our desired fuel/air mixture - x[i_fu] = (ox_di_ratio*equiv_ratio)/(stoich_ratio+ox_di_ratio*equiv_ratio) - x[i_ox] = stoich_ratio*x[i_fu]/equiv_ratio - x[i_di] = (1.0-ox_di_ratio)*x[i_ox]/ox_di_ratio - # Uncomment next line to make pylint fail when it can't find cantera.one_atm - one_atm = cantera.one_atm # pylint: disable=no-member - # one_atm = 101325.0 - - # Let the user know about how Cantera is being initilized - print(f"Input state (T,P,X) = ({init_temperature}, {one_atm}, {x}") - # Set Cantera internal gas temperature, pressure, and mole fractios - cantera_soln.TPX = init_temperature, one_atm, x - # Pull temperature, total density, mass fractions, and pressure from Cantera - # We need total density, and mass fractions to initialize the fluid/gas state. - can_t, can_rho, can_y = cantera_soln.TDY - can_p = cantera_soln.P - # *can_t*, *can_p* should not differ (significantly) from user's initial data, - # but we want to ensure that we use exactly the same starting point as Cantera, - # so we use Cantera's version of these data. - - # }}} - - # {{{ Create Pyrometheus thermochemistry object & EOS - - # Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and - # generates a set of methods to calculate chemothermomechanical properties and - # states for this particular mechanism. - casename = "mixture-adaptive" - pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np) - eos = PyrometheusMixture(pyrometheus_mechanism, - temperature_guess=init_temperature) - - # }}} - - # {{{ MIRGE-Com state initialization - - # Initialize the fluid/gas state with Cantera-consistent data: - # (density, pressure, temperature, mass_fractions) - print(f"Cantera state (rho,T,P,Y) = ({can_rho}, {can_t}, {can_p}, {can_y}") - initializer = MixtureInitializer(dim=dim, nspecies=nspecies, - pressure=can_p, temperature=can_t, - massfractions=can_y, velocity=velocity) - - my_boundary = AdiabaticSlipBoundary() - boundaries = {BTAG_ALL: my_boundary} - current_state = initializer(eos=eos, x_vec=nodes, t=0) - - # Inspection at physics debugging time - if debug: - print("Initial MIRGE-Com state:") - print(f"{current_state=}") - print(f"Initial DV pressure: {eos.pressure(current_state)}") - print(f"Initial DV temperature: {eos.temperature(current_state)}") - - # }}} - - visualizer = make_visualizer(discr) - initname = initializer.__class__.__name__ - eosname = eos.__class__.__name__ - init_message = make_init_message(dim=dim, order=order, - nelements=local_nelements, - global_nelements=global_nelements, - dt=current_dt, t_final=t_final, nstatus=nstatus, - nviz=nviz, cfl=current_cfl, - constant_cfl=constant_cfl, initname=initname, - eosname=eosname, casename=casename) - - # Cantera equilibrate calculates the expected end state @ chemical equilibrium - # i.e. the expected state after all reactions - cantera_soln.equilibrate("UV") - eq_temperature, eq_density, eq_mass_fractions = cantera_soln.TDY - eq_pressure = cantera_soln.P - - # Report the expected final state to the user - if rank == 0: - logger.info(init_message) - logger.info(f"Expected equilibrium state:" - f" {eq_pressure=}, {eq_temperature=}," - f" {eq_density=}, {eq_mass_fractions=}") - - def my_rhs(t, state): - return (euler_operator(discr, cv=state, t=t, - boundaries=boundaries, eos=eos) - + eos.get_species_source_terms(state)) - - def mixture_prestep_function(step, t, dt, state): - do_viz = check_step(step, nviz) - viz_fields = [("cv", state)] - current_dt = dt - - if constant_cfl: - local_dt = get_inviscid_timestep(discr, eos=eos, cv=state) - from grudge.op import nodal_min - current_dt = current_cfl * nodal_min(discr, "vol", local_dt) - else: # constant dt mode - if do_viz: # calculate cfl field only if visualizing - local_cfl = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state) - if do_viz: # extend viz field if viz time - # Calculate DV only if needed for visualization - dv = eos.dependent_vars(state) - viz_fields.append(("dv", dv)) - # only if vizzing, calculate reaction rates - reaction_rates = eos.get_production_rates(state) - viz_fields.append(("reaction_rates", reaction_rates)) - if constant_cfl: - viz_fields.append(("dt", local_dt)) - else: # constant dt mode - viz_fields.append(("cfl", local_cfl)) - - errors = current_dt < 0 or np.isnan(current_dt) or current_dt == np.inf - - if do_viz or errors: # write viz at viztime, or if there were errors - from mirgecom.io import make_rank_fname, make_par_fname - rank_fn = make_rank_fname(basename=casename, rank=rank, step=step, t=t) - visualizer.write_parallel_vtk_file( - comm, rank_fn, viz_fields, overwrite=True, - par_manifest_filename=make_par_fname( - basename=casename, step=step, t=t - ) - ) - - if check_step(step, nstatus) or errors: - if not do_viz: # we already have dv on viz steps - dv = eos.dependent_vars(state) - from grudge.op import nodal_max - min_temperature = nodal_min(discr, "vol", dv.temperature) - max_temperature = nodal_max(discr, "vol", dv.temperature) - min_pressure = nodal_min(discr, "vol", dv.pressure) - max_pressure = nodal_max(discr, "vol", dv.pressure) - if rank == 0: - logger.info(f"\nStep:{step}, Time:{t}, DT:{current_dt}," - f"CFL:{current_cfl}\n" - f"---- P({min_pressure}, {max_pressure})\n" - f"---- T({min_temperature}, {max_temperature})\n") - - # this exit is safe, errors(current_dt) is already collective - if errors: - logger.info("Fatal error: Invalid simulation DT") - import sys - sys.exit() - - t_remaining = max(0, t_final - t) - return min(t_remaining, current_dt) - - try: - (current_step, current_t, current_state) = \ - advance_state(rhs=my_rhs, timestepper=timestepper, - checkpoint=mixture_prestep_function, state=current_state, - t=current_t, t_final=t_final) - - except ExactSolutionMismatch as ex: - error_state = True - current_step = ex.step - current_t = ex.t - current_state = ex.state - - if not check_step(current_step, nviz): # If final step not an output step - if rank == 0: - logger.info("Checkpointing final state ...") - mixture_prestep_function(current_step, t=current_t, - dt=(current_t - checkpoint_t), - state=current_state) - - if current_t - t_final < 0: - error_state = True - - if error_state: - raise ValueError("Simulation did not complete successfully.") - - if rank == 0: - logger.info(f"Simulation finished at time {current_t=}.") - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main(use_leap=False) - -# vim: foldmethod=marker From e07d9a7fc8cd81421a8d615fa6286f31a6d875d2 Mon Sep 17 00:00:00 2001 From: Kaushik Kulkarni <15399010+kaushikcfd@users.noreply.github.com> Date: Sat, 10 Jul 2021 10:35:02 -0500 Subject: [PATCH 252/385] Tag iel, idof with Concurrent(Element|DOF)InameTag (#419) Co-authored-by: Michael Campbell --- mirgecom/artificial_viscosity.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mirgecom/artificial_viscosity.py b/mirgecom/artificial_viscosity.py index 33093b7d7..4f58a8c3c 100644 --- a/mirgecom/artificial_viscosity.py +++ b/mirgecom/artificial_viscosity.py @@ -272,7 +272,10 @@ def smoothness_indicator(discr, u, kappa=1.0, s0=-6.0): def indicator_prg(): """Compute the smoothness indicator for all elements.""" from arraycontext import make_loopy_program - return make_loopy_program([ + from meshmode.transform_metadata import (ConcurrentElementInameTag, + ConcurrentDOFInameTag) + import loopy as lp + t_unit = make_loopy_program([ "{[iel]: 0 <= iel < nelements}", "{[idof]: 0 <= idof < ndiscr_nodes_in}", "{[jdof]: 0 <= jdof < ndiscr_nodes_in}", @@ -288,6 +291,8 @@ def indicator_prg(): """, name="smooth_comp", ) + return lp.tag_inames(t_unit, {"iel": ConcurrentElementInameTag(), + "idof": ConcurrentDOFInameTag()}) @keyed_memoize_in(actx, (smoothness_indicator, "highest_mode"), From f4f49cb0c811b9c0dd3ecf6c8ec391e97a9b54af Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 10 Jul 2021 12:28:04 -0500 Subject: [PATCH 253/385] Modernize doublemach example. --- examples/doublemach-mpi.py | 302 +++++++++++++++++++++++++++---------- 1 file changed, 225 insertions(+), 77 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index f5566b982..42a22a714 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -25,6 +25,7 @@ """ import logging +import numpy as np import pyopencl as cl import pyopencl.tools as cl_tools from functools import partial @@ -43,12 +44,6 @@ av_operator, smoothness_indicator ) -from mirgecom.simutil import ( - inviscid_sim_timestep, - sim_checkpoint, - generate_and_distribute_mesh, - ExactSolutionMismatch, -) from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -62,9 +57,26 @@ from mirgecom.eos import IdealSingleGas from mirgecom.transport import SimpleTransport +from logpyle import set_dt +from mirgecom.euler import extract_vars_for_logging, units_for_logging +from mirgecom.profiling import PyOpenCLProfilingArrayContext +from mirgecom.logging_quantities import ( + initialize_logmgr, + logmgr_add_many_discretization_quantities, + logmgr_add_device_name, + logmgr_add_device_memory_usage, + set_sim_state +) + logger = logging.getLogger(__name__) +class MyRuntimeError(RuntimeError): + """Simple exception to kill the simulation.""" + + pass + + def get_doublemach_mesh(): """Generate or import a grid using `gmsh`. @@ -112,13 +124,33 @@ def get_doublemach_mesh(): @mpi_entry_point -def main(ctx_factory=cl.create_some_context): +def main(ctx_factory=cl.create_some_context, use_leap=False, + use_profiling=False, rst_step=None, rst_name=None, + casename="doubleMach", use_logmgr=True): """Drive the example.""" cl_ctx = ctx_factory() - queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext( - queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)) - ) + + if casename is None: + casename = "mirgecom" + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + nparts = comm.Get_size() + + logmgr = initialize_logmgr(use_logmgr, + filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) + + if use_profiling: + queue = cl.CommandQueue(cl_ctx, + properties=cl.command_queue_properties.PROFILING_ENABLE) + actx = PyOpenCLProfilingArrayContext(queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), + logmgr=logmgr) + else: + queue = cl.CommandQueue(cl_ctx) + actx = PyOpenCLArrayContext(queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) dim = 2 order = 3 @@ -135,7 +167,6 @@ def main(ctx_factory=cl.create_some_context): # }}} eos = IdealSingleGas(transport_model=transport_model) initializer = DoubleMachReflection() - casename = "doubleMach" boundaries = { DTAG_BOUNDARY("ic1"): PrescribedBoundary(initializer), @@ -147,33 +178,70 @@ def main(ctx_factory=cl.create_some_context): constant_cfl = False nstatus = 10 nviz = 100 - rank = 0 - checkpoint_t = current_t current_step = 0 timestepper = rk4_step + nrestart = 100 + nhealth = 1 s0 = -6.0 kappa = 1.0 alpha = 2.0e-2 from mpi4py import MPI - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - - gen_grid = partial(get_doublemach_mesh) - - local_mesh, global_nelements = generate_and_distribute_mesh(comm, gen_grid) - - local_nelements = local_mesh.nelements + rst_path = "restart_data/" + rst_pattern = ( + rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" + ) + if rst_step: # read the grid from restart data + rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) + + from mirgecom.restart import read_restart_data + restart_data = read_restart_data(actx, rst_fname) + local_mesh = restart_data["local_mesh"] + local_nelements = local_mesh.nelements + global_nelements = restart_data["global_nelements"] + assert restart_data["nparts"] == nparts + else: # generate the grid from scratch + gen_grid = partial(get_doublemach_mesh) + from mirgecom.simutil import generate_and_distribute_mesh + local_mesh, global_nelements = generate_and_distribute_mesh(comm, gen_grid) + local_nelements = local_mesh.nelements discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) - nodes = thaw(actx, discr.nodes()) - current_state = initializer(nodes) + + if logmgr: + logmgr_add_device_name(logmgr, queue) + logmgr_add_device_memory_usage(logmgr, queue) + logmgr_add_many_discretization_quantities(logmgr, discr, dim, + extract_vars_for_logging, units_for_logging) + + logmgr.add_watches([ + ("step.max", "step = {value}, "), + ("t_sim.max", "sim time: {value:1.6e} s\n"), + ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), + ("max_pressure", "{value:1.9e})\n"), + ("min_temperature", "------- T (min, max) (K) = ({value:1.9e}, "), + ("max_temperature", "{value:1.9e})\n"), + ("t_step.max", "------- step walltime: {value:6g} s, "), + ("t_log.max", "log walltime: {value:6g} s") + ]) + + if rst_step: + current_t = restart_data["t"] + current_step = rst_step + current_state = restart_data["state"] + if logmgr: + from mirgecom.logging_quantities import logmgr_set_time + logmgr_set_time(logmgr, current_step, current_t) + else: + # Set the current state from time 0 + current_state = initializer(nodes) visualizer = make_visualizer(discr, discr.order if discr.dim == 2 else discr.order) + initname = initializer.__class__.__name__ eosname = eos.__class__.__name__ init_message = make_init_message( @@ -194,16 +262,123 @@ def main(ctx_factory=cl.create_some_context): if rank == 0: logger.info(init_message) - get_timestep = partial( - inviscid_sim_timestep, - discr=discr, - t=current_t, - dt=current_dt, - cfl=current_cfl, - eos=eos, - t_final=t_final, - constant_cfl=constant_cfl, - ) + def my_write_viz(step, t, state, dv=None, tagged_cells=None): + if dv is None: + dv = eos.dependent_vars(state) + if tagged_cells is None: + tagged_cells = smoothness_indicator(discr, state.mass, s0=s0, + kappa=kappa) + viz_fields = [("cv", state), + ("dv", dv), + ("tagged_cells", tagged_cells)] + from mirgecom.simutil import write_visfile + write_visfile(discr, viz_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + def my_write_restart(step, t, state): + rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": nparts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(state, dv): + # Note: This health check is tuned s.t. it is a test that + # the case gets the expected solution. If dt,t_final or + # other run parameters are changed, this check should + # be changed accordingly. + health_error = False + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure): + health_error = True + logger.info(f"{rank=}: NANs/Infs in pressure data.") + + from mirgecom.simutil import allsync + if allsync(check_range_local(discr, "vol", dv.pressure, .9, 18.6), + comm, op=MPI.LOR): + health_error = True + from grudge.op import nodal_max, nodal_min + p_min = nodal_min(discr, "vol", dv.pressure) + p_max = nodal_max(discr, "vol", dv.pressure) + logger.info(f"Pressure range violation ({p_min=}, {p_max=})") + + if check_naninf_local(discr, "vol", dv.temperature): + health_error = True + logger.info(f"{rank=}: NANs/INFs in temperature data.") + + if allsync( + check_range_local(discr, "vol", dv.temperature, 2.48e-3, 1.071e-2), + comm, op=MPI.LOR): + health_error = True + from grudge.op import nodal_max, nodal_min + t_min = nodal_min(discr, "vol", dv.temperature) + t_max = nodal_max(discr, "vol", dv.temperature) + logger.info(f"Temperature range violation ({t_min=}, {t_max=})") + + return health_error + + def my_pre_step(step, t, dt, state): + try: + dv = None + + if logmgr: + logmgr.tick_before() + + from mirgecom.simutil import check_step + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) + + if do_health: + dv = eos.dependent_vars(state) + from mirgecom.simutil import allsync + health_errors = allsync(my_health_check(state, dv), comm, + op=MPI.LOR) + if health_errors: + if rank == 0: + logger.info("Fluid solution failed health check.") + raise MyRuntimeError("Failed simulation health check.") + + if step == rst_step: # don't do viz or restart @ restart + do_viz = False + do_restart = False + + if do_restart: + my_write_restart(step=step, t=t, state=state) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + tagged_cells = smoothness_indicator(discr, state.mass, s0=s0, + kappa=kappa) + my_write_viz(step=step, t=t, state=state, dv=dv, + tagged_cells=tagged_cells) + + except MyRuntimeError: + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + my_write_viz(step=step, t=t, state=state) + my_write_restart(step=step, t=t, state=state) + raise + + t_remaining = max(0, t_final - t) + return state, min(dt, t_remaining) + + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt def my_rhs(t, state): return ns_operator( @@ -214,53 +389,26 @@ def my_rhs(t, state): s0=s0, kappa=kappa) ) - def my_checkpoint(step, t, dt, state): - tagged_cells = smoothness_indicator(discr, state.mass, s0=s0, kappa=kappa) - viz_fields = [("tagged cells", tagged_cells)] - return sim_checkpoint( - discr, - visualizer, - eos, - cv=state, - vizname=casename, - step=step, - t=t, - dt=dt, - nstatus=nstatus, - nviz=nviz, - constant_cfl=constant_cfl, - comm=comm, - viz_fields=viz_fields, - overwrite=True, - ) - - try: - (current_step, current_t, current_state) = advance_state( - rhs=my_rhs, - timestepper=timestepper, - checkpoint=my_checkpoint, - get_timestep=get_timestep, - state=current_state, - t=current_t, - t_final=t_final, - ) - except ExactSolutionMismatch as ex: - current_step = ex.step - current_t = ex.t - current_state = ex.state + current_step, current_t, current_state = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + pre_step_callback=my_pre_step, + post_step_callback=my_post_step, dt=current_dt, + state=current_state, t=current_t, t_final=t_final) - # if current_t != checkpoint_t: + # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") - my_checkpoint( - current_step, - t=current_t, - dt=(current_t - checkpoint_t), - state=current_state, - ) + final_dv = eos.dependent_vars(current_state) + my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv) + my_write_restart(step=current_step, t=current_t, state=current_state) + + if logmgr: + logmgr.close() + elif use_profiling: + print(actx.tabulate_profiling_data()) - if current_t - t_final < 0: - raise ValueError("Simulation exited abnormally") + finish_tol = 1e-16 + assert np.abs(current_t - t_final) < finish_tol if __name__ == "__main__": From c68f1e485eb0a4f242b394564dae5546b7fb4c92 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 12 Jul 2021 09:45:48 -0500 Subject: [PATCH 254/385] Clean up, succinctify some of the interfaces, add cfl to viz interface. --- examples/autoignition-mpi.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 4848f1de1..ede8e5211 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -315,28 +315,22 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, f" {eq_density=}, {eq_mass_fractions=}") def my_write_status(dt, cfl): - if constant_cfl: - status_msg = f"------ {dt=}" - else: - status_msg = f"------ {cfl=}" + status_msg = f"------ {dt=}" if constant_cfl else f"----- {cfl=}" if rank == 0: logger.info(status_msg) def my_write_viz(step, t, dt, state, ts_field=None, dv=None, - production_rates=None): + production_rates=None, cfl=None): if dv is None: dv = eos.dependent_vars(state) if production_rates is None: production_rates = eos.get_production_rates(state) - viz_fields = [("cv", state), - ("dv", dv), - ("production_rates", production_rates)] if ts_field is None: ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=state) - if constant_cfl: - viz_fields.append(("local_dt", ts_field)) - else: - viz_fields.append(("local_cfl", ts_field)) + viz_fields = [("cv", state), + ("dv", dv), + ("production_rates", production_rates), + ("dt" if constant_cfl else "cfl", ts_field)] write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) @@ -373,6 +367,7 @@ def my_health_check(dv): return health_error def my_get_timestep(t, dt, state): + # richer interface to calculate {dt,dfl} returns node-local estimates t_remaining = max(0, t_final - t) if constant_cfl: from mirgecom.inviscid import get_inviscid_timestep @@ -424,7 +419,7 @@ def my_pre_step(step, t, dt, state): dv = eos.dependent_vars(state) my_write_viz(step=step, t=t, dt=dt, state=state, dv=dv, production_rates=production_rates, - ts_field=ts_field) + ts_field=ts_field, cfl=cfl) except MyRuntimeError: if rank == 0: @@ -464,7 +459,7 @@ def my_rhs(t, state): ts_field, cfl, dt = my_get_timestep(t=current_t, dt=current_dt, state=current_state) my_write_viz(step=current_step, t=current_t, dt=dt, state=current_state, - dv=final_dv, production_rates=final_dm, ts_field=ts_field) + dv=final_dv, production_rates=final_dm, ts_field=ts_field, cfl=cfl) my_write_status(dt=dt, cfl=cfl) my_write_restart(step=current_step, t=current_t, state=current_state) From a3250f430bbbd230643d5988be0e108e46c89d4e Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 12 Jul 2021 09:47:48 -0500 Subject: [PATCH 255/385] Rarrange cosmetically into logical sections, use simutil timestep util. --- examples/lump-mpi.py | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index 75eae69c4..f088a89e4 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -96,30 +96,26 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - dim = 3 - nel_1d = 16 - order = 3 + # timestepping control + if use_leap: + from leap.rk import RK4MethodBuilder + timestepper = RK4MethodBuilder("state") + else: + timestepper = rk4_step t_final = 0.01 current_cfl = 1.0 - vel = np.zeros(shape=(dim,)) - orig = np.zeros(shape=(dim,)) - vel[:dim] = 1.0 current_dt = .001 current_t = 0 - eos = IdealSingleGas() - initializer = Lump(dim=dim, center=orig, velocity=vel) - boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} + current_step = 0 constant_cfl = False + + # some i/o frequencies nstatus = 1 nhealth = 1 nrestart = 10 nviz = 1 - current_step = 0 - if use_leap: - from leap.rk import RK4MethodBuilder - timestepper = RK4MethodBuilder("state") - else: - timestepper = rk4_step + + dim = 3 rst_path = "restart_data/" rst_pattern = ( @@ -137,6 +133,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, else: # generate the grid from scratch box_ll = -5.0 box_ur = 5.0 + nel_1d = 16 from meshmode.mesh.generation import generate_regular_rect_mesh generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) @@ -144,6 +141,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, generate_mesh) local_nelements = local_mesh.nelements + order = 3 discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) @@ -167,6 +165,14 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) + # soln setup, init + eos = IdealSingleGas() + vel = np.zeros(shape=(dim,)) + orig = np.zeros(shape=(dim,)) + vel[:dim] = 1.0 + initializer = Lump(dim=dim, center=orig, velocity=vel) + boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} + if rst_step: current_t = restart_data["t"] current_step = rst_step @@ -299,8 +305,10 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - t_remaining = max(0, t_final - t) - return state, min(dt, t_remaining) + from mirgecom.simutil import get_sim_timestep + dt = get_sim_timestep(discr, state, t, dt, current_cfl, eos, t_final, + constant_cfl) + return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? From 5d373ac4b02301a78412008aa0e2ff73b2088f53 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 12 Jul 2021 09:49:13 -0500 Subject: [PATCH 256/385] Use simutil timestep util, clean up restart handling, some cosmetics --- examples/mixture-mpi.py | 105 +++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 50 deletions(-) diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index eb1ec13ea..0b73ed486 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -72,8 +72,8 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, rst_step=None, rst_name=None, - casename="uiuc_mixture", use_logmgr=True): + use_profiling=False, restart_file=None, casename="uiuc_mixture", + use_logmgr=True): """Drive example.""" cl_ctx = ctx_factory() @@ -99,41 +99,39 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - dim = 3 - nel_1d = 16 - order = 3 + # timestepping control + if use_leap: + from leap.rk import RK4MethodBuilder + timestepper = RK4MethodBuilder("state") + else: + timestepper = rk4_step t_final = 0.002 current_cfl = 1.0 - velocity = np.zeros(shape=(dim,)) - velocity[:dim] = 1.0 current_dt = .001 current_t = 0 + current_step = 0 constant_cfl = False + + # some i/o frequencies nstatus = 1 nhealth = 1 nrestart = 5 nviz = 1 - current_step = 0 - if use_leap: - from leap.rk import RK4MethodBuilder - timestepper = RK4MethodBuilder("state") - else: - timestepper = rk4_step + dim = 3 rst_path = "restart_data/" rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) - if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) - + if restart_file: # read the grid from restart data from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, rst_fname) + restart_data = read_restart_data(actx, restart_file) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["nparts"] == nparts else: # generate the grid from scratch + nel_1d = 16 box_ll = -5.0 box_ur = 5.0 from meshmode.mesh.generation import generate_regular_rect_mesh @@ -143,6 +141,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, generate_mesh) local_nelements = local_mesh.nelements + order = 3 discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) @@ -182,14 +181,16 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, y0s[nspecies-1] = 1.0 - spec_sum # Mixture defaults to STP (p, T) = (1atm, 300K) + velocity = np.zeros(shape=(dim,)) + velocity[:dim] = 1.0 initializer = MixtureInitializer(dim=dim, nspecies=nspecies, massfractions=y0s, velocity=velocity) boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} nodes = thaw(actx, discr.nodes()) - if rst_step: + if restart_file: current_t = restart_data["t"] - current_step = rst_step + current_step = restart_data["step"] current_state = restart_data["state"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time @@ -211,6 +212,13 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, if rank == 0: logger.info(init_message) + def my_write_status(component_errors): + status_msg = ( + "------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + if rank == 0: + logger.info(status_msg) + def my_write_viz(step, t, state, dv=None, exact=None, resid=None): viz_fields = [("cv", state)] if dv is None: @@ -229,19 +237,20 @@ def my_write_viz(step, t, state, dv=None, exact=None, resid=None): def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) - rst_data = { - "local_mesh": local_mesh, - "state": state, - "t": t, - "step": step, - "order": order, - "global_nelements": global_nelements, - "num_parts": nparts - } - from mirgecom.restart import write_restart_file - write_restart_file(actx, rst_data, rst_fname, comm) - - def my_health_check(state, dv, exact): + if rst_fname != restart_file: + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": nparts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(dv, component_errors): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure) \ @@ -249,8 +258,6 @@ def my_health_check(state, dv, exact): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") - from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact) exittol = .09 if max(component_errors) > exittol: health_error = True @@ -263,6 +270,7 @@ def my_pre_step(step, t, dt, state): try: dv = None exact = None + component_errors = None if logmgr: logmgr.tick_before() @@ -276,18 +284,16 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) from mirgecom.simutil import allsync - health_errors = allsync(my_health_check(state, dv, exact), comm, + health_errors = allsync(my_health_check(dv, component_errors), comm, op=MPI.LOR) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False - if do_restart: my_write_restart(step=step, t=t, state=state) @@ -301,15 +307,12 @@ def my_pre_step(step, t, dt, state): resid=resid) if do_status: - if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) - from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact) - status_msg = ( - "------- errors=" - + ", ".join("%.3g" % en for en in component_errors)) - if rank == 0: - logger.info(status_msg) + if component_errors is None: + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + my_write_status(component_errors) except MyRuntimeError: if rank == 0: @@ -318,8 +321,10 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - t_remaining = max(0, t_final - t) - return state, min(dt, t_remaining) + from mirgecom.simutil import get_sim_timestep + dt = get_sim_timestep(discr, state, t, dt, current_cfl, eos, t_final, + constant_cfl) + return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? From 71d74f8d9f8f1b327d82e4381131b22c2251fe40 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 12 Jul 2021 10:15:19 -0500 Subject: [PATCH 257/385] Rename the timestepping interface, add note about viscous --- mirgecom/simutil.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 6a52d80c7..591752335 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -4,7 +4,7 @@ ----------------- .. autofunction:: check_step -.. autofunction:: inviscid_sim_timestep +.. autofunction:: get_sim_timestep .. autofunction:: write_visfile .. autofunction:: allsync @@ -48,7 +48,6 @@ import logging import numpy as np -from mirgecom.inviscid import get_inviscid_timestep # bad smell? import grudge.op as op logger = logging.getLogger(__name__) @@ -75,13 +74,13 @@ def check_step(step, interval): return False -def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, - t_final, constant_cfl=False): - """Return the maximum stable dt for inviscid fluid simulation. +def get_sim_timestep(discr, state, t, dt, cfl, eos, + t_final, constant_cfl=False): + """Return the maximum stable dt for fluid simulation. This routine returns *dt*, the users defined constant timestep, or *max_dt*, the maximum domain-wide stability-limited - timestep for an inviscid fluid simulation. It calls the collective: + timestep for a fluid simulation. It calls the collective: :func:`~grudge.op.nodal_min` on the inside which makes it domain-wide regardless of parallel decomposition. @@ -89,6 +88,11 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, - Constant DT mode: returns the minimum of (t_final-t, dt) - Constant CFL mode: returns (cfl * max_dt) + .. important:: + The current implementation is calculating an acoustic-limited + timestep and CFL for an inviscid fluid. The addition of viscous + fluxes includes modification to this routine. + Parameters ---------- discr @@ -116,6 +120,7 @@ def inviscid_sim_timestep(discr, state, t, dt, cfl, eos, mydt = dt t_remaining = max(0, t_final - t) if constant_cfl: + from mirgecom.inviscid import get_inviscid_timestep from grudge.op import nodal_min mydt = cfl * nodal_min( discr, "vol", From 07fa2728ba630a0e80cdac871b50be8ec440e50c Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 12 Jul 2021 11:56:05 -0500 Subject: [PATCH 258/385] Clean up examples a bit, massage timestepping control for constant CFL mode --- examples/autoignition-mpi.py | 13 ++-- examples/lump-mpi.py | 50 +++++++------ examples/mixture-mpi.py | 20 +++-- examples/pulse-mpi.py | 88 +++++++++++----------- examples/scalar-lump-mpi.py | 139 +++++++++++++++++++---------------- examples/sod-mpi.py | 93 ++++++++++++----------- examples/vortex-mpi.py | 119 ++++++++++++++++-------------- 7 files changed, 283 insertions(+), 239 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index ede8e5211..836f97fa3 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -43,6 +43,7 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( + get_sim_timestep, generate_and_distribute_mesh, write_visfile ) @@ -143,12 +144,11 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) - restarting = rst_filename is not None - if restarting: # read the grid from restart data - rst_fname = f"{rst_filename}-{rank:04d}.pkl" + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, rst_fname) + restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] @@ -261,7 +261,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, my_boundary = AdiabaticSlipBoundary() boundaries = {BTAG_ALL: my_boundary} - if restarting: + if rst_filename: current_step = rst_step current_t = rst_time if logmgr: @@ -444,6 +444,9 @@ def my_rhs(t, state): boundaries=boundaries, eos=eos) + eos.get_species_source_terms(state)) + current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index f088a89e4..1cc4553c2 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -37,7 +37,10 @@ from mirgecom.euler import euler_operator -from mirgecom.simutil import generate_and_distribute_mesh +from mirgecom.simutil import ( + get_sim_timestep, + generate_and_distribute_mesh +) from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -69,8 +72,8 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, rst_step=None, rst_name=None, - casename="lump", use_logmgr=True): + use_profiling=False, rst_filename=None, casename="lump", + use_logmgr=True): """Drive example.""" cl_ctx = ctx_factory() @@ -121,11 +124,11 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) - if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, rst_fname) + restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] @@ -173,9 +176,9 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, initializer = Lump(dim=dim, center=orig, velocity=vel) boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} - if rst_step: + if rst_filename: current_t = restart_data["t"] - current_step = rst_step + current_step = restart_data["step"] current_state = restart_data["state"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time @@ -214,17 +217,18 @@ def my_write_viz(step, t, state, dv=None, exact=None, resid=None): def my_write_restart(state, step, t): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) - rst_data = { - "local_mesh": local_mesh, - "state": state, - "t": t, - "step": step, - "order": order, - "global_nelements": global_nelements, - "num_parts": nparts - } - from mirgecom.restart import write_restart_file - write_restart_file(actx, rst_data, rst_fname, comm) + if rst_fname != rst_filename: + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": nparts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(dv, state, exact): health_error = False @@ -271,10 +275,6 @@ def my_pre_step(step, t, dt, state): logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed solution health check.") - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False - if do_restart: my_write_restart(step=step, t=t, state=state) @@ -305,7 +305,6 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - from mirgecom.simutil import get_sim_timestep dt = get_sim_timestep(discr, state, t, dt, current_cfl, eos, t_final, constant_cfl) return state, dt @@ -323,6 +322,9 @@ def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) + current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 0b73ed486..5de084950 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -37,7 +37,10 @@ from mirgecom.euler import euler_operator -from mirgecom.simutil import generate_and_distribute_mesh +from mirgecom.simutil import ( + get_sim_timestep, + generate_and_distribute_mesh +) from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -72,7 +75,7 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, restart_file=None, casename="uiuc_mixture", + use_profiling=False, rst_filename=None, casename="uiuc_mixture", use_logmgr=True): """Drive example.""" cl_ctx = ctx_factory() @@ -123,9 +126,10 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) - if restart_file: # read the grid from restart data + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, restart_file) + restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] @@ -188,7 +192,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} nodes = thaw(actx, discr.nodes()) - if restart_file: + if rst_filename: current_t = restart_data["t"] current_step = restart_data["step"] current_state = restart_data["state"] @@ -237,7 +241,7 @@ def my_write_viz(step, t, state, dv=None, exact=None, resid=None): def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) - if rst_fname != restart_file: + if rst_fname != rst_filename: rst_data = { "local_mesh": local_mesh, "state": state, @@ -321,7 +325,6 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - from mirgecom.simutil import get_sim_timestep dt = get_sim_timestep(discr, state, t, dt, current_cfl, eos, t_final, constant_cfl) return state, dt @@ -339,6 +342,9 @@ def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) + current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 7afdda6d3..3831897b4 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -38,7 +38,10 @@ from grudge.shortcuts import make_visualizer from mirgecom.euler import euler_operator -from mirgecom.simutil import generate_and_distribute_mesh +from mirgecom.simutil import ( + get_sim_timestep, + generate_and_distribute_mesh +) from mirgecom.io import make_init_message from mirgecom.integrators import rk4_step @@ -76,7 +79,7 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename="pulse", - rst_step=None, rst_name=None): + rst_filename=None): """Drive the example.""" cl_ctx = ctx_factory() @@ -102,41 +105,34 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - dim = 2 - nel_1d = 16 - order = 1 + # timestepping control + current_step = 0 + if use_leap: + from leap.rk import RK4MethodBuilder + timestepper = RK4MethodBuilder("state") + else: + timestepper = rk4_step t_final = 0.1 current_cfl = 1.0 - vel = np.zeros(shape=(dim,)) - orig = np.zeros(shape=(dim,)) current_dt = .01 current_t = 0 - eos = IdealSingleGas() - initializer = Lump(dim=dim, center=orig, velocity=vel, rhoamp=0.0) - boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} - wall = AdiabaticSlipBoundary() - boundaries = {BTAG_ALL: wall} constant_cfl = False + + # some i/o frequencies nstatus = 1 nrestart = 5 nviz = 10 nhealth = 1 - current_step = 0 - if use_leap: - from leap.rk import RK4MethodBuilder - timestepper = RK4MethodBuilder("state") - else: - timestepper = rk4_step + dim = 2 rst_path = "restart_data/" rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) - if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) - + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, rst_fname) + restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] @@ -145,12 +141,14 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, from meshmode.mesh.generation import generate_regular_rect_mesh box_ll = -0.5 box_ur = 0.5 + nel_1d = 16 generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) local_nelements = local_mesh.nelements + order = 1 discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) @@ -171,12 +169,19 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, ("t_log.max", "log walltime: {value:6g} s") ]) + eos = IdealSingleGas() + vel = np.zeros(shape=(dim,)) + orig = np.zeros(shape=(dim,)) + initializer = Lump(dim=dim, center=orig, velocity=vel, rhoamp=0.0) + boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} + wall = AdiabaticSlipBoundary() + boundaries = {BTAG_ALL: wall} uniform_state = initializer(nodes) acoustic_pulse = AcousticPulse(dim=dim, amplitude=1.0, width=.1, center=orig) - if rst_step: + if rst_filename: current_t = restart_data["t"] - current_step = rst_step + current_step = restart_data["step"] current_state = restart_data["state"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time @@ -210,17 +215,18 @@ def my_write_viz(step, t, state, dv=None): def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) - rst_data = { - "local_mesh": local_mesh, - "state": state, - "t": t, - "step": step, - "order": order, - "global_nelements": global_nelements, - "num_parts": num_parts - } - from mirgecom.restart import write_restart_file - write_restart_file(actx, rst_data, rst_fname, comm) + if rst_fname != rst_filename: + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": num_parts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(pressure): health_error = False @@ -253,10 +259,6 @@ def my_pre_step(step, t, dt, state): logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False - if do_restart: my_write_restart(step=step, t=t, state=state) @@ -272,8 +274,9 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - t_remaining = max(0, t_final - t) - return state, min(dt, t_remaining) + dt = get_sim_timestep(discr, state, t, dt, current_cfl, eos, t_final, + constant_cfl) + return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? @@ -288,6 +291,9 @@ def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) + current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index c7ce42fdc..067ff22ea 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -38,7 +38,10 @@ from mirgecom.euler import euler_operator -from mirgecom.simutil import generate_and_distribute_mesh +from mirgecom.simutil import ( + get_sim_timestep, + generate_and_distribute_mesh +) from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -70,7 +73,7 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, rst_step=None, rst_name=None, + use_profiling=False, rst_filename=None, casename="lumpy-scalars", use_logmgr=True): """Drive example.""" cl_ctx = ctx_factory() @@ -97,50 +100,34 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - dim = 3 - nel_1d = 16 - order = 3 + # timestepping control + current_step = 0 + if use_leap: + from leap.rk import RK4MethodBuilder + timestepper = RK4MethodBuilder("state") + else: + timestepper = rk4_step t_final = 0.005 current_cfl = 1.0 current_dt = .001 current_t = 0 constant_cfl = False + + # some i/o frequencies nstatus = 1 nrestart = 5 nviz = 1 nhealth = 1 - current_step = 0 - if use_leap: - from leap.rk import RK4MethodBuilder - timestepper = RK4MethodBuilder("state") - else: - timestepper = rk4_step - box_ll = -5.0 - box_ur = 5.0 - - nspecies = 4 - centers = make_obj_array([np.zeros(shape=(dim,)) for i in range(nspecies)]) - spec_y0s = np.ones(shape=(nspecies,)) - spec_amplitudes = np.ones(shape=(nspecies,)) - eos = IdealSingleGas() - - velocity = np.ones(shape=(dim,)) - - initializer = MulticomponentLump(dim=dim, nspecies=nspecies, - spec_centers=centers, velocity=velocity, - spec_y0s=spec_y0s, - spec_amplitudes=spec_amplitudes) - boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} + dim = 3 rst_path = "restart_data/" rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) - if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) - + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, rst_fname) + restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] @@ -148,6 +135,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, else: # generate the grid from scratch box_ll = -5.0 box_ur = 5.0 + nel_1d = 16 from meshmode.mesh.generation import generate_regular_rect_mesh generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) @@ -155,6 +143,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, generate_mesh) local_nelements = local_mesh.nelements + order = 3 discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) @@ -178,9 +167,22 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) - if rst_step: + # soln setup and init + nspecies = 4 + centers = make_obj_array([np.zeros(shape=(dim,)) for i in range(nspecies)]) + spec_y0s = np.ones(shape=(nspecies,)) + spec_amplitudes = np.ones(shape=(nspecies,)) + eos = IdealSingleGas() + velocity = np.ones(shape=(dim,)) + + initializer = MulticomponentLump(dim=dim, nspecies=nspecies, + spec_centers=centers, velocity=velocity, + spec_y0s=spec_y0s, + spec_amplitudes=spec_amplitudes) + boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} + if rst_filename: current_t = restart_data["t"] - current_step = rst_step + current_step = restart_data["step"] current_state = restart_data["state"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time @@ -202,6 +204,12 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, if rank == 0: logger.info(init_message) + def my_write_status(component_errors): + if rank == 0: + logger.info( + "------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: dv = eos.dependent_vars(state) @@ -219,19 +227,20 @@ def my_write_viz(step, t, state, dv=None, exact=None, resid=None): def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) - rst_data = { - "local_mesh": local_mesh, - "state": state, - "t": t, - "step": step, - "order": order, - "global_nelements": global_nelements, - "num_parts": nparts - } - from mirgecom.restart import write_restart_file - write_restart_file(actx, rst_data, rst_fname, comm) - - def my_health_check(state, pressure, exact): + if rst_fname != rst_filename: + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": nparts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(pressure, component_errors): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", pressure) \ @@ -239,8 +248,6 @@ def my_health_check(state, pressure, exact): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") - from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact) exittol = .09 if max(component_errors) > exittol: health_error = True @@ -253,6 +260,7 @@ def my_pre_step(step, t, dt, state): try: dv = None exact = None + component_errors = None if logmgr: logmgr.tick_before() @@ -266,18 +274,18 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) from mirgecom.simutil import allsync - health_errors = allsync(my_health_check(state, dv.pressure, exact), - comm, op=MPI.LOR) + health_errors = allsync( + my_health_check(dv.pressure, component_errors), + comm, op=MPI.LOR + ) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False - if do_restart: my_write_restart(step=step, t=t, state=state) @@ -291,15 +299,12 @@ def my_pre_step(step, t, dt, state): resid=resid) if do_status: - if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) - from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact) - status_msg = ( - "------- errors=" - + ", ".join("%.3g" % en for en in component_errors)) - if rank == 0: - logger.info(status_msg) + if component_errors is None: + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + my_write_status(component_errors) except MyRuntimeError: if rank == 0: @@ -308,8 +313,9 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - t_remaining = max(0, t_final - t) - return state, min(dt, t_remaining) + dt = get_sim_timestep(discr, state, t, dt, current_cfl, eos, t_final, + constant_cfl) + return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? @@ -324,6 +330,9 @@ def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) + current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, dt=current_dt, diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 9e3165f4e..888a68627 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -37,7 +37,10 @@ from mirgecom.euler import euler_operator -from mirgecom.simutil import generate_and_distribute_mesh +from mirgecom.simutil import ( + get_sim_timestep, + generate_and_distribute_mesh +) from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point @@ -70,7 +73,7 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=True, use_leap=False, use_profiling=False, casename="sod1d", - rst_step=None, rst_name=None): + rst_filename=None): """Drive the example.""" cl_ctx = ctx_factory() @@ -97,7 +100,6 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) dim = 1 - nel_1d = 24 order = 1 # tolerate large errors; case is unstable t_final = 0.01 @@ -123,17 +125,17 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) - if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) - + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, rst_fname) + restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["nparts"] == num_parts else: # generate the grid from scratch from meshmode.mesh.generation import generate_regular_rect_mesh + nel_1d = 24 box_ll = -5.0 box_ur = 5.0 generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, @@ -162,9 +164,9 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, ("t_log.max", "log walltime: {value:6g} s") ]) - if rst_step: + if rst_filename: current_t = restart_data["t"] - current_step = rst_step + current_step = restart_data["step"] current_state = restart_data["state"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time @@ -187,6 +189,13 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, if rank == 0: logger.info(init_message) + def my_write_status(component_errors): + if rank == 0: + logger.info( + "------- errors=" + + ", ".join("%.3g" % en for en in component_errors) + ) + def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: dv = eos.dependent_vars(state) @@ -204,19 +213,20 @@ def my_write_viz(step, t, state, dv=None, exact=None, resid=None): def my_write_restart(state, step, t): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) - rst_data = { - "local_mesh": local_mesh, - "state": state, - "t": t, - "step": step, - "order": order, - "global_nelements": global_nelements, - "num_parts": num_parts - } - from mirgecom.restart import write_restart_file - write_restart_file(actx, rst_data, rst_fname, comm) - - def my_health_check(state, pressure, exact): + if rst_fname != rst_filename: + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": num_parts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(pressure, component_errors): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", pressure) \ @@ -224,8 +234,6 @@ def my_health_check(state, pressure, exact): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") - from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact) exittol = .09 if max(component_errors) > exittol: health_error = True @@ -238,6 +246,7 @@ def my_pre_step(step, t, dt, state): try: dv = None exact = None + component_errors = None if logmgr: logmgr.tick_before() @@ -251,18 +260,18 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) from mirgecom.simutil import allsync - health_errors = allsync(my_health_check(state, dv.pressure, exact), - comm, op=MPI.LOR) + health_errors = allsync( + my_health_check(dv.pressure, exact, component_errors), + comm, op=MPI.LOR + ) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False - if do_restart: my_write_restart(step=step, t=t, state=state) @@ -276,15 +285,13 @@ def my_pre_step(step, t, dt, state): resid=resid) if do_status: - if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) - from mirgecom.simutil import compare_fluid_solutions - component_errors = compare_fluid_solutions(discr, state, exact) - status_msg = ( - "------- errors=" - + ", ".join("%.3g" % en for en in component_errors)) - if rank == 0: - logger.info(status_msg) + if component_errors is None: + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) + from mirgecom.simutil import compare_fluid_solutions + component_errors = \ + compare_fluid_solutions(discr, state, exact) + my_write_status(component_errors) except MyRuntimeError: if rank == 0: @@ -293,8 +300,9 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - t_remaining = max(0, t_final - t) - return state, min(dt, t_remaining) + dt = get_sim_timestep(discr, state, t, dt, current_cfl, eos, t_final, + constant_cfl) + return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? @@ -309,6 +317,9 @@ def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) + current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, dt=current_dt, diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 4831d0c0c..356f5c99b 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -39,6 +39,7 @@ from mirgecom.euler import euler_operator from mirgecom.simutil import ( + get_sim_timestep, generate_and_distribute_mesh, check_step ) @@ -73,8 +74,8 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=True, - use_leap=False, use_profiling=False, casename="sod1d", - rst_step=None, rst_name=None): + use_leap=False, use_profiling=False, rst_filename=None, + casename="vortex"): """Drive the example.""" cl_ctx = ctx_factory() @@ -100,32 +101,26 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - dim = 2 - nel_1d = 16 - order = 3 + # timestepping control + current_step = 0 + if use_leap: + from leap.rk import RK4MethodBuilder + timestepper = RK4MethodBuilder("state") + else: + timestepper = rk4_step t_final = 0.01 current_cfl = 1.0 - vel = np.zeros(shape=(dim,)) - orig = np.zeros(shape=(dim,)) - vel[:dim] = 1.0 current_dt = .001 current_t = 0 - eos = IdealSingleGas() - initializer = Vortex2D(center=orig, velocity=vel) - casename = "vortex" - boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} constant_cfl = False + + # some i/o frequencies nrestart = 10 nstatus = 1 nviz = 10 nhealth = 10 - current_step = 0 - if use_leap: - from leap.rk import RK4MethodBuilder - timestepper = RK4MethodBuilder("state") - else: - timestepper = rk4_step + dim = 2 if dim != 2: raise ValueError("This example must be run with dim = 2.") @@ -133,25 +128,26 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) - if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) - + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, rst_fname) + restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["nparts"] == num_parts else: # generate the grid from scratch - from meshmode.mesh.generation import generate_regular_rect_mesh + nel_1d = 16 box_ll = -5.0 box_ur = 5.0 + from meshmode.mesh.generation import generate_regular_rect_mesh generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,)*dim, b=(box_ur,) * dim, nelements_per_axis=(nel_1d,)*dim) local_mesh, global_nelements = generate_and_distribute_mesh(comm, generate_mesh) local_nelements = local_mesh.nelements + order = 3 discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) @@ -185,9 +181,16 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, vis_timer = IntervalTimer("t_vis", "Time spent visualizing") logmgr.add_quantity(vis_timer) - if rst_step: + # soln setup and init + eos = IdealSingleGas() + vel = np.zeros(shape=(dim,)) + orig = np.zeros(shape=(dim,)) + vel[:dim] = 1.0 + initializer = Vortex2D(center=orig, velocity=vel) + boundaries = {BTAG_ALL: PrescribedBoundary(initializer)} + if rst_filename: current_t = restart_data["t"] - current_step = rst_step + current_step = restart_data["step"] current_state = restart_data["state"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time @@ -210,15 +213,16 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, if rank == 0: logger.info(init_message) - def my_write_status(component_errors, cfl=None): - status_msg = "" - if cfl is not None: - status_msg = f"------ {cfl=}\n" - status_msg += ( - "------- errors=" - + ", ".join("%.3g" % en for en in component_errors)) + def my_write_status(state, component_errors, cfl=None): + if cfl is None: + from mirgecom.inviscid import get_inviscid_cfl + cfl = current_cfl if constant_cfl else \ + get_inviscid_cfl(discr, eos, current_dt, state) if rank == 0: - logger.info(status_msg) + logger.info( + f"------ {cfl=}\n" + "------- errors=" + + ", ".join("%.3g" % en for en in component_errors)) def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: @@ -237,19 +241,20 @@ def my_write_viz(step, t, state, dv=None, exact=None, resid=None): def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) - rst_data = { - "local_mesh": local_mesh, - "state": state, - "t": t, - "step": step, - "order": order, - "global_nelements": global_nelements, - "num_parts": num_parts - } - from mirgecom.restart import write_restart_file - write_restart_file(actx, rst_data, rst_fname, comm) - - def my_health_check(state, pressure, exact, component_errors): + if rst_fname != rst_filename: + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": num_parts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(pressure, component_errors): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", pressure) \ @@ -269,6 +274,7 @@ def my_pre_step(step, t, dt, state): try: dv = None exact = None + component_errors = None if logmgr: logmgr.tick_before() @@ -285,7 +291,7 @@ def my_pre_step(step, t, dt, state): component_errors = compare_fluid_solutions(discr, state, exact) from mirgecom.simutil import allsync health_errors = allsync( - my_health_check(state, dv.pressure, exact, component_errors), + my_health_check(dv.pressure, component_errors), comm, op=MPI.LOR ) if health_errors: @@ -293,19 +299,16 @@ def my_pre_step(step, t, dt, state): logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False - if do_restart: my_write_restart(step=step, t=t, state=state) if do_status: - if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + if component_errors is None: + if exact is None: + exact = initializer(x_vec=nodes, eos=eos, t=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, state, exact) - my_write_status(component_errors) + my_write_status(state, component_errors) if do_viz: if dv is None: @@ -323,8 +326,9 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - t_remaining = max(0, t_final - t) - return state, min(dt, t_remaining) + dt = get_sim_timestep(discr, state, t, dt, current_cfl, eos, t_final, + constant_cfl) + return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? @@ -339,6 +343,9 @@ def my_rhs(t, state): return euler_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) + current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, From 59fc7a495143028ec79a0185afc06334064a7b15 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 12 Jul 2021 12:03:33 -0500 Subject: [PATCH 259/385] Correct health call. --- examples/sod-mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 888a68627..2e4bb350f 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -264,7 +264,7 @@ def my_pre_step(step, t, dt, state): component_errors = compare_fluid_solutions(discr, state, exact) from mirgecom.simutil import allsync health_errors = allsync( - my_health_check(dv.pressure, exact, component_errors), + my_health_check(dv.pressure, component_errors), comm, op=MPI.LOR ) if health_errors: From 4595fccc8079fd2230feb5afdb6cae6705fd4232 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Mon, 12 Jul 2021 17:04:09 -0500 Subject: [PATCH 260/385] Sharpen and correct docstrings. Co-authored-by: Thomas H. Gibson --- examples/autoignition-mpi.py | 2 +- mirgecom/inviscid.py | 2 +- mirgecom/simutil.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 836f97fa3..965054130 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -367,7 +367,7 @@ def my_health_check(dv): return health_error def my_get_timestep(t, dt, state): - # richer interface to calculate {dt,dfl} returns node-local estimates + # richer interface to calculate {dt,cfl} returns node-local estimates t_remaining = max(0, t_final - t) if constant_cfl: from mirgecom.inviscid import get_inviscid_timestep diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index b14d2cad3..29a7bc9fc 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -67,7 +67,7 @@ def inviscid_flux(discr, eos, cv): def get_inviscid_timestep(discr, eos, cv): - """Routine returns the node-local maximum stable dt for inviscid fluid. + """Returns node-local estimates of the maximum stable timestep size for an inviscid fluid. The maximum stable timestep is computed from the acoustic wavespeed. diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 591752335..3b0a03c1e 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -76,7 +76,7 @@ def check_step(step, interval): def get_sim_timestep(discr, state, t, dt, cfl, eos, t_final, constant_cfl=False): - """Return the maximum stable dt for fluid simulation. + """Return the maximum stable timestep for a typical fluid simulation. This routine returns *dt*, the users defined constant timestep, or *max_dt*, the maximum domain-wide stability-limited From 2fe594aad657913293df0b2817c5efd122b95beb Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Mon, 12 Jul 2021 17:08:10 -0500 Subject: [PATCH 261/385] Shorten the line without perturbing the info? --- mirgecom/inviscid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 29a7bc9fc..e3fe348d2 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -67,7 +67,7 @@ def inviscid_flux(discr, eos, cv): def get_inviscid_timestep(discr, eos, cv): - """Returns node-local estimates of the maximum stable timestep size for an inviscid fluid. + """Return node-local stable timestep estimate for an inviscid fluid. The maximum stable timestep is computed from the acoustic wavespeed. @@ -94,7 +94,7 @@ def get_inviscid_timestep(discr, eos, cv): def get_inviscid_cfl(discr, eos, dt, cv): - """Calculate and return node-local CFL based on current state and timestep. + """Return node-local CFL based on current state and timestep. Parameters ---------- From 69a5fa480d1daca5d09d826b4a66ca3e9b5d56cf Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 12 Jul 2021 18:06:24 -0500 Subject: [PATCH 262/385] Update production setup to be compatible with CFL/DT development. --- .ci-support/production-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci-support/production-setup.sh b/.ci-support/production-setup.sh index df547769a..d2b739782 100644 --- a/.ci-support/production-setup.sh +++ b/.ci-support/production-setup.sh @@ -1,8 +1,8 @@ #!/bin/bash set -x -PRODUCTION_CHANGE_OWNER="illinois-ceesd" -PRODUCTION_CHANGE_BRANCH="" +PRODUCTION_CHANGE_OWNER="MTCam" +PRODUCTION_CHANGE_BRANCH="y1-update-cfl-dt" CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) git config user.email "stupid@dumb.com" git config user.name "CI Runner" From b32d7e649977b0c379038fe798c6fa77a84e6b2e Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 13 Jul 2021 05:19:20 -0500 Subject: [PATCH 263/385] Bring up-to-date with CFL/DT --- examples/doublemach-mpi.py | 103 +++++++++++++++++++------------------ examples/nsmix-mpi.py | 63 ++++++++++++----------- examples/poiseuille-mpi.py | 62 +++++++++++----------- 3 files changed, 118 insertions(+), 110 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 44dcc6932..21756bbd4 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -55,6 +55,7 @@ from mirgecom.initializers import DoubleMachReflection from mirgecom.eos import IdealSingleGas from mirgecom.transport import SimpleTransport +from mirgecom.simutil import get_sim_timestep from logpyle import set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging @@ -124,7 +125,7 @@ def get_doublemach_mesh(): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, rst_step=None, rst_name=None, + use_profiling=False, rst_filename=None, casename="doubleMach", use_logmgr=True): """Drive the example.""" cl_ctx = ctx_factory() @@ -151,51 +152,30 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - dim = 2 - order = 3 - # Too many steps for CI - # t_final = 1.0e-2 + # Timestepping control + current_step = 0 + timestepper = rk4_step t_final = 1.0e-3 current_cfl = 0.1 current_dt = 1.0e-4 current_t = 0 - # {{{ Initialize simple transport model - kappa = 1e-5 - sigma = 1e-5 - transport_model = SimpleTransport(viscosity=sigma, thermal_conductivity=kappa) - # }}} - eos = IdealSingleGas(transport_model=transport_model) - initializer = DoubleMachReflection() - - boundaries = { - DTAG_BOUNDARY("ic1"): PrescribedBoundary(initializer), - DTAG_BOUNDARY("ic2"): PrescribedBoundary(initializer), - DTAG_BOUNDARY("ic3"): PrescribedBoundary(initializer), - DTAG_BOUNDARY("wall"): AdiabaticNoslipMovingBoundary(), - DTAG_BOUNDARY("out"): AdiabaticNoslipMovingBoundary(), - } constant_cfl = False + + # Some i/o frequencies nstatus = 10 nviz = 100 - current_step = 0 - timestepper = rk4_step nrestart = 100 nhealth = 1 - s0 = -6.0 - kappa = 1.0 - alpha = 2.0e-2 - from mpi4py import MPI - rst_path = "restart_data/" rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) - if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, rst_fname) + restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] @@ -206,10 +186,12 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, local_mesh, global_nelements = generate_and_distribute_mesh(comm, gen_grid) local_nelements = local_mesh.nelements + order = 3 discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) nodes = thaw(actx, discr.nodes()) + dim = 2 if logmgr: logmgr_add_device_name(logmgr, queue) logmgr_add_device_memory_usage(logmgr, queue) @@ -227,9 +209,30 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, ("t_log.max", "log walltime: {value:6g} s") ]) - if rst_step: + # Solution setup and initialization + s0 = -6.0 + kappa = 1.0 + alpha = 2.0e-2 + # {{{ Initialize simple transport model + kappa_t = 1e-5 + sigma_v = 1e-5 + transport_model = SimpleTransport(viscosity=sigma_v, + thermal_conductivity=kappa_t) + # }}} + eos = IdealSingleGas(transport_model=transport_model) + initializer = DoubleMachReflection() + + boundaries = { + DTAG_BOUNDARY("ic1"): PrescribedBoundary(initializer), + DTAG_BOUNDARY("ic2"): PrescribedBoundary(initializer), + DTAG_BOUNDARY("ic3"): PrescribedBoundary(initializer), + DTAG_BOUNDARY("wall"): AdiabaticNoslipMovingBoundary(), + DTAG_BOUNDARY("out"): AdiabaticNoslipMovingBoundary(), + } + + if rst_filename: current_t = restart_data["t"] - current_step = rst_step + current_step = restart_data["step"] current_state = restart_data["state"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time @@ -276,17 +279,18 @@ def my_write_viz(step, t, state, dv=None, tagged_cells=None): def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) - rst_data = { - "local_mesh": local_mesh, - "state": state, - "t": t, - "step": step, - "order": order, - "global_nelements": global_nelements, - "num_parts": nparts - } - from mirgecom.restart import write_restart_file - write_restart_file(actx, rst_data, rst_fname, comm) + if rst_fname != rst_filename: + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": nparts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(state, dv): # Note: This health check is tuned s.t. it is a test that @@ -345,10 +349,6 @@ def my_pre_step(step, t, dt, state): logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False - if do_restart: my_write_restart(step=step, t=t, state=state) @@ -367,8 +367,10 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - t_remaining = max(0, t_final - t) - return state, min(dt, t_remaining) + dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + + return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? @@ -388,6 +390,9 @@ def my_rhs(t, state): s0=s0, kappa=kappa) ) + current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, diff --git a/examples/nsmix-mpi.py b/examples/nsmix-mpi.py index 1a9fb75c2..96ad8c8d8 100644 --- a/examples/nsmix-mpi.py +++ b/examples/nsmix-mpi.py @@ -75,7 +75,7 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, rst_step=None, rst_name=None, + use_profiling=False, rst_filename=None, casename="nsmix", use_logmgr=True): """Drive example.""" cl_ctx = ctx_factory() @@ -102,41 +102,39 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - dim = 2 - nel_1d = 8 - order = 1 - + # Timestepping control # This example runs only 3 steps by default (to keep CI ~short) - # With the mixture defined below, equilibrium is achieved at ~40ms - # To run to equlibrium, set t_final >= 40ms. t_final = 3e-9 current_cfl = .0009 - velocity = np.zeros(shape=(dim,)) current_dt = 1e-9 current_t = 0 constant_cfl = True + current_step = 0 + timestepper = rk4_step + debug = False + + # Some i/o frequencies nstatus = 1 nviz = 5 nrestart = 5 nhealth = 1 - current_step = 0 - timestepper = rk4_step - debug = False + dim = 2 rst_path = "restart_data/" rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) - if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, rst_fname) + restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["nparts"] == nparts else: # generate the grid from scratch + nel_1d = 8 box_ll = -0.005 box_ur = 0.005 from meshmode.mesh.generation import generate_regular_rect_mesh @@ -147,6 +145,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, generate_mesh) local_nelements = local_mesh.nelements + order = 1 discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) @@ -239,6 +238,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, # }}} # {{{ MIRGE-Com state initialization + velocity = np.zeros(shape=(dim,)) # Initialize the fluid/gas state with Cantera-consistent data: # (density, pressure, temperature, mass_fractions) @@ -251,9 +251,9 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, my_boundary = IsothermalNoSlipBoundary(wall_temperature=can_t) visc_bnds = {BTAG_ALL: my_boundary} - if rst_step: + if rst_filename: current_t = restart_data["t"] - current_step = rst_step + current_step = restart_data["step"] current_state = restart_data["state"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time @@ -325,17 +325,18 @@ def my_write_viz(step, t, state, dv=None, production_rates=None): def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) - rst_data = { - "local_mesh": local_mesh, - "state": state, - "t": t, - "step": step, - "order": order, - "global_nelements": global_nelements, - "num_parts": nparts - } - from mirgecom.restart import write_restart_file - write_restart_file(actx, rst_data, rst_fname, comm) + if rst_fname != rst_filename: + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": nparts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(state, dv): # Note: This health check is tuned to expected results @@ -395,10 +396,6 @@ def my_pre_step(step, t, dt, state): logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False - if do_restart: my_write_restart(step=step, t=t, state=state) @@ -438,6 +435,10 @@ def my_rhs(t, state): reaction_source = eos.get_species_source_terms(state) return ns_rhs + reaction_source + current_dt = get_sim_timestep(discr, current_state, current_t, + current_dt, current_cfl, eos, + t_final, constant_cfl) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, diff --git a/examples/poiseuille-mpi.py b/examples/poiseuille-mpi.py index 78f458b5b..3c1bc0d22 100644 --- a/examples/poiseuille-mpi.py +++ b/examples/poiseuille-mpi.py @@ -87,7 +87,7 @@ def _get_box_mesh(dim, a, b, n, t=None): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, rst_step=None, rst_name=None, + use_profiling=False, rst_filename=None, casename="poiseuille", use_logmgr=True): """Drive the example.""" cl_ctx = ctx_factory() @@ -114,42 +114,43 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, actx = PyOpenCLArrayContext(queue, allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - dim = 2 - order = 1 + # timestepping control + timestepper = rk4_step t_final = 1e-6 current_cfl = 0.1 current_dt = 1e-8 current_t = 0 - casename = "poiseuille" constant_cfl = True + current_step = 0 + + # some i/o frequencies nstatus = 1 nviz = 1 nrestart = 100 nhealth = 1 - current_step = 0 - timestepper = rk4_step - left_boundary_location = 0 - right_boundary_location = 0.1 - npts_axis = (50, 30) - rank = comm.Get_rank() + # some geometry setup + dim = 2 if dim != 2: raise ValueError("This example must be run with dim = 2.") + left_boundary_location = 0 + right_boundary_location = 0.1 rst_path = "restart_data/" rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) - if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, rst_fname) + restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] assert restart_data["nparts"] == nparts else: # generate the grid from scratch + npts_axis = (50, 30) box_ll = (left_boundary_location, 0.0) box_ur = (right_boundary_location, 0.02) generate_mesh = partial(_get_box_mesh, 2, a=box_ll, b=box_ur, n=npts_axis) @@ -158,6 +159,7 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, generate_mesh) local_nelements = local_mesh.nelements + order = 1 discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) @@ -212,9 +214,9 @@ def poiseuille_soln(nodes, eos, cv=None, **kwargs): DTAG_BOUNDARY("+2"): IsothermalNoSlipBoundary()} eos = IdealSingleGas(transport_model=SimpleTransport(viscosity=1.0)) - if rst_step: + if rst_filename: current_t = restart_data["t"] - current_step = rst_step + current_step = restart_data["step"] current_state = restart_data["state"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time @@ -266,17 +268,18 @@ def my_write_viz(step, t, state, dv=None, exact=None): def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) - rst_data = { - "local_mesh": local_mesh, - "state": state, - "t": t, - "step": step, - "order": order, - "global_nelements": global_nelements, - "num_parts": nparts - } - from mirgecom.restart import write_restart_file - write_restart_file(actx, rst_data, rst_fname, comm) + if rst_fname != rst_filename: + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": nparts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(state, dv): health_error = False @@ -331,10 +334,6 @@ def my_pre_step(step, t, dt, state): logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False - if do_restart: my_write_restart(step=step, t=t, state=state) @@ -372,6 +371,9 @@ def my_post_step(step, t, dt, state): def my_rhs(t, state): return ns_operator(discr, eos=eos, boundaries=boundaries, cv=state, t=t) + current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, From 2fe8e905f9d7d15ee9ab7e129c0f5734d9da011d Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 13 Jul 2021 05:43:39 -0500 Subject: [PATCH 264/385] Update CFL reporting - report nodal max --- examples/vortex-mpi.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index 8b33a4ab2..b5cf40c42 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -217,9 +217,13 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, def my_write_status(state, component_errors, cfl=None): if cfl is None: - from mirgecom.inviscid import get_inviscid_cfl - cfl = current_cfl if constant_cfl else \ - get_inviscid_cfl(discr, eos, current_dt, state) + if constant_cfl: + cfl = current_cfl + else: + from grudge.op import nodal_max + from mirgecom.inviscid import get_inviscid_cfl + cfl = nodal_max(discr, "vol", + get_inviscid_cfl(discr, eos, current_dt, cv=state)) if rank == 0: logger.info( f"------ {cfl=}\n" From 8024d2bdab98da123a2893d59ac9736762f28fcb Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 13 Jul 2021 05:54:18 -0500 Subject: [PATCH 265/385] Sync with cfl/dt --- .ci-support/production-setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci-support/production-setup.sh b/.ci-support/production-setup.sh index d2b739782..73e7c30dc 100644 --- a/.ci-support/production-setup.sh +++ b/.ci-support/production-setup.sh @@ -1,8 +1,8 @@ #!/bin/bash set -x -PRODUCTION_CHANGE_OWNER="MTCam" -PRODUCTION_CHANGE_BRANCH="y1-update-cfl-dt" +PRODUCTION_CHANGE_OWNER="" +PRODUCTION_CHANGE_BRANCH="" CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) git config user.email "stupid@dumb.com" git config user.name "CI Runner" From c4f8419a68359262c6e1f9a2ad730a3601a20f6a Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 13 Jul 2021 05:56:43 -0500 Subject: [PATCH 266/385] Add gitattributes --- .ci-support/.gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .ci-support/.gitattributes diff --git a/.ci-support/.gitattributes b/.ci-support/.gitattributes new file mode 100644 index 000000000..d7496e23b --- /dev/null +++ b/.ci-support/.gitattributes @@ -0,0 +1,2 @@ +production-setup.sh merge=ours +production-driver-setup.sh merge=ours From 493767e45f4310f636940a2501a3265fe58e340f Mon Sep 17 00:00:00 2001 From: Matthew Smith Date: Wed, 14 Jul 2021 13:24:30 -0500 Subject: [PATCH 267/385] fix CURRENT_FORK_OWNER --- .ci-support/production-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-setup.sh b/.ci-support/production-setup.sh index 3d59a6259..2541be443 100644 --- a/.ci-support/production-setup.sh +++ b/.ci-support/production-setup.sh @@ -15,7 +15,7 @@ if [ -n "${PRODUCTION_CHANGE_BRANCH}" ]; then else echo "No updates to production branch (${CURRENT_BRANCH})" fi -CURRENT_FORK_OWNER="majosm" +CURRENT_FORK_OWNER="" if [ -n "${GITHUB_HEAD_REF}" ]; then git remote add changes https://github.com/${CURRENT_FORK_OWNER}/mirgecom git fetch changes From fc278bbd8d54ec637b06c3a59389c80c3388010f Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 14 Jul 2021 13:39:02 -0500 Subject: [PATCH 268/385] Move the CURRENT_FORK_OWNER back to illinois-ceesd --- .ci-support/production-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-setup.sh b/.ci-support/production-setup.sh index 2541be443..73e7c30dc 100644 --- a/.ci-support/production-setup.sh +++ b/.ci-support/production-setup.sh @@ -15,7 +15,7 @@ if [ -n "${PRODUCTION_CHANGE_BRANCH}" ]; then else echo "No updates to production branch (${CURRENT_BRANCH})" fi -CURRENT_FORK_OWNER="" +CURRENT_FORK_OWNER="illinois-ceesd" if [ -n "${GITHUB_HEAD_REF}" ]; then git remote add changes https://github.com/${CURRENT_FORK_OWNER}/mirgecom git fetch changes From 257f18d8bdc784cc348cf8cb8e9d923a723afd76 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 14 Jul 2021 14:24:39 -0500 Subject: [PATCH 269/385] Update production testing setup to be slightly more robust against wicked merges that try to change them. --- .ci-support/production-setup.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.ci-support/production-setup.sh b/.ci-support/production-setup.sh index 3d59a6259..750d49b11 100644 --- a/.ci-support/production-setup.sh +++ b/.ci-support/production-setup.sh @@ -7,6 +7,9 @@ CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) git config user.email "stupid@dumb.com" git config user.name "CI Runner" if [ -n "${PRODUCTION_CHANGE_BRANCH}" ]; then + if [ -z "${PRODUCTION_CHANGE_OWNER}"]; then + PRODUCTION_CHANGE_OWNER="illinois-ceesd" + fi git remote add production_change https://github.com/${PRODUCTION_CHANGE_OWNER}/mirgecom git fetch production_change git checkout production_change/${PRODUCTION_CHANGE_BRANCH} @@ -15,8 +18,11 @@ if [ -n "${PRODUCTION_CHANGE_BRANCH}" ]; then else echo "No updates to production branch (${CURRENT_BRANCH})" fi -CURRENT_FORK_OWNER="majosm" +CURRENT_FORK_OWNER="" if [ -n "${GITHUB_HEAD_REF}" ]; then + if [ -z "${CURRENT_FORK_OWNER}"]; then + CURRENT_FORK_OWNER="illinois-ceesd" + fi git remote add changes https://github.com/${CURRENT_FORK_OWNER}/mirgecom git fetch changes git checkout changes/${GITHUB_HEAD_REF} From d92fc90ee5a17a1b1f82f515ffa564a0cc90ebee Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 14 Jul 2021 14:50:14 -0500 Subject: [PATCH 270/385] update production testing scripts --- .ci-support/production-setup.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.ci-support/production-setup.sh b/.ci-support/production-setup.sh index 73e7c30dc..750d49b11 100644 --- a/.ci-support/production-setup.sh +++ b/.ci-support/production-setup.sh @@ -7,6 +7,9 @@ CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) git config user.email "stupid@dumb.com" git config user.name "CI Runner" if [ -n "${PRODUCTION_CHANGE_BRANCH}" ]; then + if [ -z "${PRODUCTION_CHANGE_OWNER}"]; then + PRODUCTION_CHANGE_OWNER="illinois-ceesd" + fi git remote add production_change https://github.com/${PRODUCTION_CHANGE_OWNER}/mirgecom git fetch production_change git checkout production_change/${PRODUCTION_CHANGE_BRANCH} @@ -15,8 +18,11 @@ if [ -n "${PRODUCTION_CHANGE_BRANCH}" ]; then else echo "No updates to production branch (${CURRENT_BRANCH})" fi -CURRENT_FORK_OWNER="illinois-ceesd" +CURRENT_FORK_OWNER="" if [ -n "${GITHUB_HEAD_REF}" ]; then + if [ -z "${CURRENT_FORK_OWNER}"]; then + CURRENT_FORK_OWNER="illinois-ceesd" + fi git remote add changes https://github.com/${CURRENT_FORK_OWNER}/mirgecom git fetch changes git checkout changes/${GITHUB_HEAD_REF} From 5c631f2480a38ab1a2da5fb8f6f922a92d51cfb6 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 14 Jul 2021 15:16:29 -0500 Subject: [PATCH 271/385] Retain my (y1) copy of the production testing script - forever. --- .ci-support/production-setup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci-support/production-setup.sh b/.ci-support/production-setup.sh index 750d49b11..c226837eb 100644 --- a/.ci-support/production-setup.sh +++ b/.ci-support/production-setup.sh @@ -27,5 +27,7 @@ if [ -n "${GITHUB_HEAD_REF}" ]; then git fetch changes git checkout changes/${GITHUB_HEAD_REF} git checkout ${CURRENT_BRANCH} + cp .ci-support/production-setup.sh my-setup.sh git merge changes/${GITHUB_HEAD_REF} --no-edit + cp my-setup.sh .ci-support/production-setup.sh fi From d61e435a6cf514dc7f57f9fa336f78e80d9fabb3 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 15 Jul 2021 15:27:55 -0500 Subject: [PATCH 272/385] Use y1-production proper --- .ci-support/production-testing-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 75d6682ef..091a1cdc5 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -15,7 +15,7 @@ set -x # patched by the incoming development. The following vars control the # production environment: # -export PRODUCTION_BRANCH="y1-production-testing" # The base production branch to be installed by emirge +# export PRODUCTION_BRANCH="y1-production-testing" # The base production branch to be installed by emirge # export PRODUCTION_CHANGE_FORK="" # The fork/home of production changes (if any) # export PRODUCTION_CHANGE_BRANCH="" # Branch from which to pull prod changes (if any) # From ac3b155df6e24592ac4a3d3a075347aba90b8285 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 15 Jul 2021 15:35:16 -0500 Subject: [PATCH 273/385] Use y1 proper --- .ci-support/production-testing-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 091a1cdc5..e64e87c8e 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -15,7 +15,7 @@ set -x # patched by the incoming development. The following vars control the # production environment: # -# export PRODUCTION_BRANCH="y1-production-testing" # The base production branch to be installed by emirge +# export PRODUCTION_BRANCH="" # The base production branch to be installed by emirge # export PRODUCTION_CHANGE_FORK="" # The fork/home of production changes (if any) # export PRODUCTION_CHANGE_BRANCH="" # Branch from which to pull prod changes (if any) # From eb19e957b91eeb26b782de8f0ee33c7787c8f44f Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 15 Jul 2021 15:55:26 -0500 Subject: [PATCH 274/385] Use y1-memoize branch for production --- .ci-support/production-testing-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index e64e87c8e..2b9d64366 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -15,7 +15,7 @@ set -x # patched by the incoming development. The following vars control the # production environment: # -# export PRODUCTION_BRANCH="" # The base production branch to be installed by emirge +export PRODUCTION_BRANCH="y1-memoize" # The base production branch to be installed by emirge # export PRODUCTION_CHANGE_FORK="" # The fork/home of production changes (if any) # export PRODUCTION_CHANGE_BRANCH="" # Branch from which to pull prod changes (if any) # From 4005f6fa7323f2117872c22763b428bfc16eafc2 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 15 Jul 2021 17:28:52 -0500 Subject: [PATCH 275/385] Use better names for the getters. --- mirgecom/eos.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mirgecom/eos.py b/mirgecom/eos.py index 142b6017a..cdfcf1ea0 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -398,11 +398,11 @@ def pressure(self, cv: ConservedVars): """ @memoize_in(cv, (PyrometheusMixture.pressure, type(self._pyrometheus_mech))) - def get(): + def get_pressure(): temperature = self.temperature(cv) y = self.species_fractions(cv) return self._pyrometheus_mech.get_pressure(cv.mass, temperature, y) - return get() + return get_pressure() def sound_speed(self, cv: ConservedVars): r"""Get the speed of sound in the gas. @@ -415,10 +415,10 @@ def sound_speed(self, cv: ConservedVars): """ @memoize_in(cv, (PyrometheusMixture.sound_speed, type(self._pyrometheus_mech))) - def get(): + def get_sos(): actx = cv.array_context return actx.np.sqrt((self.gamma(cv) * self.pressure(cv)) / cv.mass) - return get() + return get_sos() def temperature(self, cv: ConservedVars): r"""Get the thermodynamic temperature of the gas. @@ -433,12 +433,12 @@ def temperature(self, cv: ConservedVars): """ @memoize_in(cv, (PyrometheusMixture.temperature, type(self._pyrometheus_mech))) - def get(): + def get_temp(): y = self.species_fractions(cv) e = self.internal_energy(cv) / cv.mass return self._pyrometheus_mech.get_temperature(e, self._tguess, y, True) - return get() + return get_temp() def total_energy(self, cv, pressure): r""" From daf98f110ff0560dc498e44c2b325711be2b5b12 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 15 Jul 2021 20:18:35 -0500 Subject: [PATCH 276/385] Use y1 proper --- .ci-support/production-testing-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 2b9d64366..e64e87c8e 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -15,7 +15,7 @@ set -x # patched by the incoming development. The following vars control the # production environment: # -export PRODUCTION_BRANCH="y1-memoize" # The base production branch to be installed by emirge +# export PRODUCTION_BRANCH="" # The base production branch to be installed by emirge # export PRODUCTION_CHANGE_FORK="" # The fork/home of production changes (if any) # export PRODUCTION_CHANGE_BRANCH="" # Branch from which to pull prod changes (if any) # From 3b5d5f95e44efe5fe29d2961fbf2ce828fa613f5 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Sat, 17 Jul 2021 14:23:37 -0500 Subject: [PATCH 277/385] Laxify the examples in production (#441) --- examples/autoignition-mpi.py | 57 +++++++++++++++++------- examples/doublemach-mpi.py | 58 ++++++++++++++++++------ examples/heat-source-mpi.py | 61 ++++++++++++++++++++------ examples/lump-mpi.py | 53 +++++++++++++++++----- examples/mixture-mpi.py | 57 ++++++++++++++++++------ examples/nsmix-mpi.py | 59 +++++++++++++++++++------ examples/poiseuille-mpi.py | 60 ++++++++++++++++++------- examples/pulse-mpi.py | 56 ++++++++++++++++++------ examples/scalar-lump-mpi.py | 57 ++++++++++++++++++------ examples/sod-mpi.py | 85 +++++++++++++++++++++++++----------- examples/vortex-mpi.py | 60 +++++++++++++++++-------- mirgecom/boundary.py | 2 +- 12 files changed, 498 insertions(+), 167 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 1d224a4fc..ec91c9ef0 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -29,7 +29,11 @@ import pyopencl.tools as cl_tools from functools import partial -from meshmode.array_context import PyOpenCLArrayContext +from meshmode.array_context import ( + PyOpenCLArrayContext, + PytatoPyOpenCLArrayContext +) +from mirgecom.profiling import PyOpenCLProfilingArrayContext from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa @@ -39,7 +43,6 @@ from logpyle import IntervalTimer, set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging -from mirgecom.profiling import PyOpenCLProfilingArrayContext from mirgecom.euler import euler_operator from mirgecom.simutil import ( @@ -76,9 +79,9 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_logmgr=False, - use_leap=False, use_profiling=False, casename="autoignition", - rst_filename=None): +def main(ctx_factory=cl.create_some_context, use_logmgr=True, + use_leap=False, use_profiling=False, casename=None, + rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive example.""" cl_ctx = ctx_factory() @@ -96,13 +99,12 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=False, if use_profiling: queue = cl.CommandQueue(cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) - actx = PyOpenCLProfilingArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), - logmgr=logmgr) else: queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) # Some discretization parameters dim = 2 @@ -478,13 +480,36 @@ def my_rhs(t, state): if __name__ == "__main__": - logging.basicConfig(format="%(message)s", level=logging.INFO) - use_profiling = False - use_logging = True - use_leap = False + import argparse casename = "autoignition" + parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") + parser.add_argument("--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("--log", action="store_true", default=True, + help="turn on logging") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--restart_file", help="root name of restart file") + parser.add_argument("--casename", help="casename to use for i/o") + args = parser.parse_args() + if args.profiling: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext - main(use_profiling=use_profiling, use_logmgr=use_logging, use_leap=use_leap, - casename=casename) + logging.basicConfig(format="%(message)s", level=logging.INFO) + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling, + casename=casename, rst_filename=rst_filename, actx_class=actx_class) # vim: foldmethod=marker diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 21756bbd4..dc3669377 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -30,7 +30,12 @@ import pyopencl.tools as cl_tools from functools import partial -from meshmode.array_context import PyOpenCLArrayContext +from meshmode.array_context import ( + PyOpenCLArrayContext, + PytatoPyOpenCLArrayContext +) +from mirgecom.profiling import PyOpenCLProfilingArrayContext + from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.dof_desc import DTAG_BOUNDARY @@ -59,7 +64,6 @@ from logpyle import set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging -from mirgecom.profiling import PyOpenCLProfilingArrayContext from mirgecom.logging_quantities import ( initialize_logmgr, logmgr_add_many_discretization_quantities, @@ -124,9 +128,9 @@ def get_doublemach_mesh(): @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, rst_filename=None, - casename="doubleMach", use_logmgr=True): +def main(ctx_factory=cl.create_some_context, use_logmgr=True, + use_leap=False, use_profiling=False, casename=None, + rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive the example.""" cl_ctx = ctx_factory() @@ -142,15 +146,14 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: - queue = cl.CommandQueue(cl_ctx, - properties=cl.command_queue_properties.PROFILING_ENABLE) - actx = PyOpenCLProfilingArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), - logmgr=logmgr) + queue = cl.CommandQueue( + cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) # Timestepping control current_step = 0 @@ -416,7 +419,36 @@ def my_rhs(t, state): if __name__ == "__main__": + import argparse + casename = "doublemach" + parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") + parser.add_argument("--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("--log", action="store_true", default=True, + help="turn on logging") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--restart_file", help="root name of restart file") + parser.add_argument("--casename", help="casename to use for i/o") + args = parser.parse_args() + if args.profiling: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext + logging.basicConfig(format="%(message)s", level=logging.INFO) - main() + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling, + casename=casename, rst_filename=rst_filename, actx_class=actx_class) # vim: foldmethod=marker diff --git a/examples/heat-source-mpi.py b/examples/heat-source-mpi.py index 66643596d..4a2651e26 100644 --- a/examples/heat-source-mpi.py +++ b/examples/heat-source-mpi.py @@ -1,3 +1,5 @@ +"""Demonstrate heat source example.""" + __copyright__ = "Copyright (C) 2020 University of Illinois Board of Trustees" __license__ = """ @@ -25,8 +27,10 @@ import numpy.linalg as la # noqa import pyopencl as cl -from meshmode.array_context import thaw, PyOpenCLArrayContext - +from meshmode.array_context import ( + PyOpenCLArrayContext, + PytatoPyOpenCLArrayContext +) from mirgecom.profiling import PyOpenCLProfilingArrayContext from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa @@ -50,7 +54,10 @@ @mpi_entry_point -def main(use_profiling=False, use_logmgr=False): +def main(ctx_factory=cl.create_some_context, use_logmgr=True, + use_leap=False, use_profiling=False, casename=None, + rst_filename=None, actx_class=PyOpenCLArrayContext): + """Run the example.""" cl_ctx = cl.create_some_context() queue = cl.CommandQueue(cl_ctx) @@ -62,15 +69,14 @@ def main(use_profiling=False, use_logmgr=False): filename="heat-source.sqlite", mode="wu", mpi_comm=comm) if use_profiling: - queue = cl.CommandQueue(cl_ctx, - properties=cl.command_queue_properties.PROFILING_ENABLE) - actx = PyOpenCLProfilingArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), - logmgr=logmgr) + queue = cl.CommandQueue( + cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) from meshmode.distributed import MPIMeshDistributor, get_partition_by_pymetis mesh_dist = MPIMeshDistributor(comm) @@ -118,6 +124,7 @@ def main(use_profiling=False, use_logmgr=False): source_width = 0.2 + from meshmode.array_context import thaw nodes = thaw(actx, discr.nodes()) boundaries = { @@ -176,10 +183,36 @@ def rhs(t, u): if __name__ == "__main__": + import argparse + casename = "heat-source" + parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") + parser.add_argument("--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("--log", action="store_true", default=True, + help="turn on logging") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--restart_file", help="root name of restart file") + parser.add_argument("--casename", help="casename to use for i/o") + args = parser.parse_args() + if args.profiling: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext + logging.basicConfig(format="%(message)s", level=logging.INFO) - # Turn off profiling to not overwhelm CI - use_profiling = False - use_logging = True - main(use_profiling=use_profiling, use_logmgr=use_logging) + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling, + casename=casename, rst_filename=rst_filename, actx_class=actx_class) # vim: foldmethod=marker diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index c9c223def..bf5d68b05 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -29,7 +29,11 @@ import pyopencl.tools as cl_tools from functools import partial -from meshmode.array_context import PyOpenCLArrayContext +from meshmode.array_context import ( + PyOpenCLArrayContext, + PytatoPyOpenCLArrayContext +) +from mirgecom.profiling import PyOpenCLProfilingArrayContext from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.eager import EagerDGDiscretization @@ -52,7 +56,6 @@ from logpyle import IntervalTimer, set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging -from mirgecom.profiling import PyOpenCLProfilingArrayContext from mirgecom.logging_quantities import ( initialize_logmgr, logmgr_add_many_discretization_quantities, @@ -71,9 +74,9 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, rst_filename=None, casename="lump", - use_logmgr=True): +def main(ctx_factory=cl.create_some_context, use_logmgr=True, + use_leap=False, use_profiling=False, casename=None, + rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive example.""" cl_ctx = ctx_factory() @@ -91,13 +94,12 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, if use_profiling: queue = cl.CommandQueue(cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) - actx = PyOpenCLProfilingArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), - logmgr=logmgr) else: queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) # timestepping control if use_leap: @@ -356,7 +358,36 @@ def my_rhs(t, state): if __name__ == "__main__": + import argparse + casename = "mass-lump" + parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") + parser.add_argument("--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("--log", action="store_true", default=True, + help="turn on logging") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--restart_file", help="root name of restart file") + parser.add_argument("--casename", help="casename to use for i/o") + args = parser.parse_args() + if args.profiling: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext + logging.basicConfig(format="%(message)s", level=logging.INFO) - main(use_leap=False) + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling, + casename=casename, rst_filename=rst_filename, actx_class=actx_class) # vim: foldmethod=marker diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 0f8a5abdf..777c62fd4 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -29,7 +29,11 @@ import pyopencl.tools as cl_tools from functools import partial -from meshmode.array_context import PyOpenCLArrayContext +from meshmode.array_context import ( + PyOpenCLArrayContext, + PytatoPyOpenCLArrayContext +) +from mirgecom.profiling import PyOpenCLProfilingArrayContext from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.eager import EagerDGDiscretization @@ -55,7 +59,6 @@ from logpyle import IntervalTimer, set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging -from mirgecom.profiling import PyOpenCLProfilingArrayContext from mirgecom.logging_quantities import ( initialize_logmgr, logmgr_add_many_discretization_quantities, @@ -74,9 +77,9 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, rst_filename=None, casename="uiuc_mixture", - use_logmgr=True): +def main(ctx_factory=cl.create_some_context, use_logmgr=True, + use_leap=False, use_profiling=False, casename=None, + rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive example.""" cl_ctx = ctx_factory() @@ -92,15 +95,14 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: - queue = cl.CommandQueue(cl_ctx, - properties=cl.command_queue_properties.PROFILING_ENABLE) - actx = PyOpenCLProfilingArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), - logmgr=logmgr) + queue = cl.CommandQueue( + cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) # timestepping control if use_leap: @@ -376,7 +378,36 @@ def my_rhs(t, state): if __name__ == "__main__": + import argparse + casename = "uiuc-mixture" + parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") + parser.add_argument("--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("--log", action="store_true", default=True, + help="turn on logging") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--restart_file", help="root name of restart file") + parser.add_argument("--casename", help="casename to use for i/o") + args = parser.parse_args() + if args.profiling: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext + logging.basicConfig(format="%(message)s", level=logging.INFO) - main(use_leap=False) + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling, + casename=casename, rst_filename=rst_filename, actx_class=actx_class) # vim: foldmethod=marker diff --git a/examples/nsmix-mpi.py b/examples/nsmix-mpi.py index 96ad8c8d8..a83e0cacb 100644 --- a/examples/nsmix-mpi.py +++ b/examples/nsmix-mpi.py @@ -29,7 +29,11 @@ import pyopencl.tools as cl_tools from functools import partial -from meshmode.array_context import PyOpenCLArrayContext +from meshmode.array_context import ( + PyOpenCLArrayContext, + PytatoPyOpenCLArrayContext +) +from mirgecom.profiling import PyOpenCLProfilingArrayContext from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.eager import EagerDGDiscretization @@ -55,7 +59,6 @@ from logpyle import IntervalTimer, set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging -from mirgecom.profiling import PyOpenCLProfilingArrayContext from mirgecom.logging_quantities import ( initialize_logmgr, logmgr_add_many_discretization_quantities, @@ -74,9 +77,9 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, rst_filename=None, - casename="nsmix", use_logmgr=True): +def main(ctx_factory=cl.create_some_context, use_logmgr=True, + use_leap=False, use_profiling=False, casename=None, + rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive example.""" cl_ctx = ctx_factory() @@ -92,15 +95,14 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: - queue = cl.CommandQueue(cl_ctx, - properties=cl.command_queue_properties.PROFILING_ENABLE) - actx = PyOpenCLProfilingArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), - logmgr=logmgr) + queue = cl.CommandQueue( + cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) # Timestepping control # This example runs only 3 steps by default (to keep CI ~short) @@ -465,7 +467,36 @@ def my_rhs(t, state): if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - main() + import argparse + casename = "nsmix" + parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") + parser.add_argument("--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("--log", action="store_true", default=True, + help="turn on logging") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--restart_file", help="root name of restart file") + parser.add_argument("--casename", help="casename to use for i/o") + args = parser.parse_args() + if args.profiling: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext + + logging.basicConfig(format="%(message)s", level=logging.INFO) + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling, + casename=casename, rst_filename=rst_filename, actx_class=actx_class) # vim: foldmethod=marker diff --git a/examples/poiseuille-mpi.py b/examples/poiseuille-mpi.py index 3c1bc0d22..d33fe5b59 100644 --- a/examples/poiseuille-mpi.py +++ b/examples/poiseuille-mpi.py @@ -30,7 +30,11 @@ from pytools.obj_array import make_obj_array from functools import partial -from meshmode.array_context import PyOpenCLArrayContext +from meshmode.array_context import ( + PyOpenCLArrayContext, + PytatoPyOpenCLArrayContext +) +from mirgecom.profiling import PyOpenCLProfilingArrayContext from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa @@ -55,7 +59,6 @@ from logpyle import IntervalTimer, set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging -from mirgecom.profiling import PyOpenCLProfilingArrayContext from mirgecom.logging_quantities import ( initialize_logmgr, logmgr_add_many_discretization_quantities, @@ -86,9 +89,9 @@ def _get_box_mesh(dim, a, b, n, t=None): @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, rst_filename=None, - casename="poiseuille", use_logmgr=True): +def main(ctx_factory=cl.create_some_context, use_logmgr=True, + use_leap=False, use_profiling=False, casename=None, + rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive the example.""" cl_ctx = ctx_factory() @@ -104,15 +107,14 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: - queue = cl.CommandQueue(cl_ctx, - properties=cl.command_queue_properties.PROFILING_ENABLE) - actx = PyOpenCLProfilingArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), - logmgr=logmgr) + queue = cl.CommandQueue( + cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) # timestepping control timestepper = rk4_step @@ -401,10 +403,36 @@ def my_rhs(t, state): if __name__ == "__main__": - logging.basicConfig(format="%(message)s", level=logging.INFO) - use_profiling = False - use_logging = False + import argparse + casename = "poiseuille" + parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") + parser.add_argument("--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("--log", action="store_true", default=True, + help="turn on logging") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--restart_file", help="root name of restart file") + parser.add_argument("--casename", help="casename to use for i/o") + args = parser.parse_args() + if args.profiling: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext - main(use_profiling=use_profiling, use_logmgr=use_logging) + logging.basicConfig(format="%(message)s", level=logging.INFO) + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling, + casename=casename, rst_filename=rst_filename, actx_class=actx_class) # vim: foldmethod=marker diff --git a/examples/pulse-mpi.py b/examples/pulse-mpi.py index 5ee724cdc..5bd20266b 100644 --- a/examples/pulse-mpi.py +++ b/examples/pulse-mpi.py @@ -31,7 +31,11 @@ import pyopencl as cl import pyopencl.tools as cl_tools -from meshmode.array_context import PyOpenCLArrayContext +from meshmode.array_context import ( + PyOpenCLArrayContext, + PytatoPyOpenCLArrayContext +) +from mirgecom.profiling import PyOpenCLProfilingArrayContext from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.eager import EagerDGDiscretization @@ -55,7 +59,6 @@ from logpyle import IntervalTimer, set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging -from mirgecom.profiling import PyOpenCLProfilingArrayContext from mirgecom.logging_quantities import ( initialize_logmgr, logmgr_add_many_discretization_quantities, @@ -75,8 +78,8 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=True, - use_leap=False, use_profiling=False, casename="pulse", - rst_filename=None): + use_leap=False, use_profiling=False, casename=None, + rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive the example.""" cl_ctx = ctx_factory() @@ -92,15 +95,14 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: - queue = cl.CommandQueue(cl_ctx, - properties=cl.command_queue_properties.PROFILING_ENABLE) - actx = PyOpenCLProfilingArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), - logmgr=logmgr) + queue = cl.CommandQueue( + cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) # timestepping control current_step = 0 @@ -318,8 +320,36 @@ def my_rhs(t, state): if __name__ == "__main__": - logging.basicConfig(format="%(message)s", level=logging.INFO) + import argparse + casename = "pulse" + parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") + parser.add_argument("--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("--log", action="store_true", default=True, + help="turn on logging") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--restart_file", help="root name of restart file") + parser.add_argument("--casename", help="casename to use for i/o") + args = parser.parse_args() + if args.profiling: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext - main(use_leap=False) + logging.basicConfig(format="%(message)s", level=logging.INFO) + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling, + casename=casename, rst_filename=rst_filename, actx_class=actx_class) # vim: foldmethod=marker diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 93f760fa9..0185d8457 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -30,7 +30,11 @@ from functools import partial from pytools.obj_array import make_obj_array -from meshmode.array_context import PyOpenCLArrayContext +from meshmode.array_context import ( + PyOpenCLArrayContext, + PytatoPyOpenCLArrayContext +) +from mirgecom.profiling import PyOpenCLProfilingArrayContext from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.eager import EagerDGDiscretization @@ -53,7 +57,6 @@ from logpyle import IntervalTimer, set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging -from mirgecom.profiling import PyOpenCLProfilingArrayContext from mirgecom.logging_quantities import ( initialize_logmgr, logmgr_add_many_discretization_quantities, @@ -72,9 +75,9 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, rst_filename=None, - casename="lumpy-scalars", use_logmgr=True): +def main(ctx_factory=cl.create_some_context, use_logmgr=True, + use_leap=False, use_profiling=False, casename=None, + rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive example.""" cl_ctx = ctx_factory() @@ -90,15 +93,14 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: - queue = cl.CommandQueue(cl_ctx, - properties=cl.command_queue_properties.PROFILING_ENABLE) - actx = PyOpenCLProfilingArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), - logmgr=logmgr) + queue = cl.CommandQueue( + cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) # timestepping control current_step = 0 @@ -365,7 +367,36 @@ def my_rhs(t, state): if __name__ == "__main__": + import argparse + casename = "lumpy-scalars" + parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") + parser.add_argument("--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("--log", action="store_true", default=True, + help="turn on logging") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--restart_file", help="root name of restart file") + parser.add_argument("--casename", help="casename to use for i/o") + args = parser.parse_args() + if args.profiling: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext + logging.basicConfig(format="%(message)s", level=logging.INFO) - main(use_leap=False) + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling, + casename=casename, rst_filename=rst_filename, actx_class=actx_class) # vim: foldmethod=marker diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 062bfeb18..0d7dbbef7 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -29,7 +29,11 @@ import pyopencl.tools as cl_tools from functools import partial -from meshmode.array_context import PyOpenCLArrayContext +from meshmode.array_context import ( + PyOpenCLArrayContext, + PytatoPyOpenCLArrayContext +) +from mirgecom.profiling import PyOpenCLProfilingArrayContext from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.eager import EagerDGDiscretization @@ -52,7 +56,6 @@ from logpyle import IntervalTimer, set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging -from mirgecom.profiling import PyOpenCLProfilingArrayContext from mirgecom.logging_quantities import ( initialize_logmgr, logmgr_add_many_discretization_quantities, @@ -72,8 +75,8 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=True, - use_leap=False, use_profiling=False, casename="sod1d", - rst_filename=None): + use_leap=False, use_profiling=False, casename=None, + rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive the example.""" cl_ctx = ctx_factory() @@ -89,40 +92,35 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: - queue = cl.CommandQueue(cl_ctx, - properties=cl.command_queue_properties.PROFILING_ENABLE) - actx = PyOpenCLProfilingArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), - logmgr=logmgr) + queue = cl.CommandQueue( + cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - dim = 1 - order = 1 - # tolerate large errors; case is unstable + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + # timestepping control + if use_leap: + from leap.rk import RK4MethodBuilder + timestepper = RK4MethodBuilder("state") + else: + timestepper = rk4_step t_final = 0.01 current_cfl = 1.0 current_dt = .0001 current_t = 0 - eos = IdealSingleGas() - initializer = SodShock1D(dim=dim) - boundaries = { - BTAG_ALL: PrescribedInviscidBoundary(fluid_solution_func=initializer) - } constant_cfl = False + current_step = 0 + + # some i/o frequencies nstatus = 10 nrestart = 5 nviz = 10 nhealth = 10 - current_step = 0 - if use_leap: - from leap.rk import RK4MethodBuilder - timestepper = RK4MethodBuilder("state") - else: - timestepper = rk4_step + dim = 1 rst_path = "restart_data/" rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" @@ -146,6 +144,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, generate_mesh) local_nelements = local_mesh.nelements + order = 1 discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) @@ -171,6 +170,11 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, ("t_log.max", "log walltime: {value:6g} s") ]) + initializer = SodShock1D(dim=dim) + eos = IdealSingleGas() + boundaries = { + BTAG_ALL: PrescribedInviscidBoundary(fluid_solution_func=initializer) + } if rst_filename: current_t = restart_data["t"] current_step = restart_data["step"] @@ -354,7 +358,36 @@ def my_rhs(t, state): if __name__ == "__main__": + import argparse + casename = "sod-shock" + parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") + parser.add_argument("--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("--log", action="store_true", default=True, + help="turn on logging") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--restart_file", help="root name of restart file") + parser.add_argument("--casename", help="casename to use for i/o") + args = parser.parse_args() + if args.profiling: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext + logging.basicConfig(format="%(message)s", level=logging.INFO) - main(use_leap=False) + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling, + casename=casename, rst_filename=rst_filename, actx_class=actx_class) # vim: foldmethod=marker diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index d7552cca6..b631c0e95 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -29,14 +29,16 @@ import pyopencl.tools as cl_tools from functools import partial -from meshmode.array_context import PyOpenCLArrayContext +from meshmode.array_context import ( + PyOpenCLArrayContext, + PytatoPyOpenCLArrayContext +) +from mirgecom.profiling import PyOpenCLProfilingArrayContext from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.eager import EagerDGDiscretization from grudge.shortcuts import make_visualizer -from mirgecom.profiling import PyOpenCLProfilingArrayContext - from mirgecom.euler import euler_operator from mirgecom.simutil import ( get_sim_timestep, @@ -74,8 +76,8 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point def main(ctx_factory=cl.create_some_context, use_logmgr=True, - use_leap=False, use_profiling=False, rst_filename=None, - casename="vortex"): + use_leap=False, use_profiling=False, casename=None, + rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive the example.""" cl_ctx = ctx_factory() @@ -91,15 +93,14 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: - queue = cl.CommandQueue(cl_ctx, - properties=cl.command_queue_properties.PROFILING_ENABLE) - actx = PyOpenCLProfilingArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), - logmgr=logmgr) + queue = cl.CommandQueue( + cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) # timestepping control current_step = 0 @@ -380,11 +381,36 @@ def my_rhs(t, state): if __name__ == "__main__": - logging.basicConfig(format="%(message)s", level=logging.INFO) - use_profiling = False - use_logging = True - use_leap = False + import argparse + casename = "vortex" + parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") + parser.add_argument("--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("--log", action="store_true", default=True, + help="turn on logging") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--restart_file", help="root name of restart file") + parser.add_argument("--casename", help="casename to use for i/o") + args = parser.parse_args() + if args.profiling: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext - main(use_profiling=use_profiling, use_logmgr=use_logging, use_leap=use_leap) + logging.basicConfig(format="%(message)s", level=logging.INFO) + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling, + casename=casename, rst_filename=rst_filename, actx_class=actx_class) # vim: foldmethod=marker diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 3603ce767..274f10eaf 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -394,7 +394,7 @@ def adiabatic_slip_pair(self, discr, cv, btag, **kwargs): def exterior_grad_q(self, nodes, nhat, grad_cv, **kwargs): """Get the exterior grad(Q) on the boundary.""" # Grab some boundary-relevant data - num_equations, dim = grad_cv.mass.shape + dim, = grad_cv.mass.shape # Subtract 2*wall-normal component of q # to enforce q=0 on the wall From 019470324af7456a2b6b80aa31fb49ea8697c336 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 17 Jul 2021 19:39:10 -0500 Subject: [PATCH 278/385] Use array context comparison in init --- mirgecom/initializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index b167bec33..009fbce92 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -254,7 +254,7 @@ def __call__(self, x_vec, *, eos=None, **kwargs): x0 = zeros + self._x0 energyl = zeros + gmn1 * self._energyl energyr = zeros + gmn1 * self._energyr - yesno = x_rel > x0 + yesno = actx.np.greater(x_rel, x0) mass = actx.np.where(yesno, rhor, rhol) energy = actx.np.where(yesno, energyr, energyl) mom = make_obj_array( From f1cd59151bd931ec5dd1a7ad7711fcc9c7364e35 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 20 Jul 2021 13:37:44 -0500 Subject: [PATCH 279/385] Pick up VTK requirement --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index b0654c66a..2a95e69ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ importlib-resources psutil gmsh PyYAML +vtk # The following packages will be git cloned by emirge: --editable git+https://github.com/inducer/pymbolic.git#egg=pymbolic From 20dd0b7742ed68a0e8138f584a9e064e63ca6c7b Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 20 Jul 2021 20:33:22 -0500 Subject: [PATCH 280/385] Bring back time arg to init --- examples/lump-mpi.py | 10 +++++----- examples/mixture-mpi.py | 10 +++++----- examples/scalar-lump-mpi.py | 10 +++++----- examples/sod-mpi.py | 10 +++++----- examples/vortex-mpi.py | 10 +++++----- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/examples/lump-mpi.py b/examples/lump-mpi.py index bf5d68b05..1a7d0d4d8 100644 --- a/examples/lump-mpi.py +++ b/examples/lump-mpi.py @@ -210,7 +210,7 @@ def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: dv = eos.dependent_vars(state) if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) if resid is None: resid = state - exact viz_fields = [("cv", state), @@ -270,7 +270,7 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import allsync health_errors = allsync( my_health_check(dv=dv, state=state, exact=exact), @@ -288,14 +288,14 @@ def my_pre_step(step, t, dt, state): if dv is None: dv = eos.dependent_vars(state) if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) resid = state - exact my_write_viz(step=step, t=t, dv=dv, state=state, exact=exact, resid=resid) if do_status: if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, state, exact) status_msg = ( @@ -342,7 +342,7 @@ def my_rhs(t, state): logger.info("Checkpointing final state ...") final_dv = eos.dependent_vars(current_state) - final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) + final_exact = initializer(x_vec=nodes, eos=eos, time=current_t) final_resid = current_state - final_exact my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, exact=final_exact, resid=final_resid) diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 777c62fd4..0bd2dbd9c 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -234,7 +234,7 @@ def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: dv = eos.dependent_vars(state) if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) if resid is None: resid = state - exact viz_fields = [("cv", state), @@ -293,7 +293,7 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, state, exact) from mirgecom.simutil import allsync @@ -311,7 +311,7 @@ def my_pre_step(step, t, dt, state): if dv is None: dv = eos.dependent_vars(state) if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) resid = state - exact my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, resid=resid) @@ -319,7 +319,7 @@ def my_pre_step(step, t, dt, state): if do_status: if component_errors is None: if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, state, exact) my_write_status(component_errors) @@ -362,7 +362,7 @@ def my_rhs(t, state): if rank == 0: logger.info("Checkpointing final state ...") final_dv = eos.dependent_vars(current_state) - final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) + final_exact = initializer(x_vec=nodes, eos=eos, time=current_t) final_resid = current_state - final_exact my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, exact=final_exact, resid=final_resid) diff --git a/examples/scalar-lump-mpi.py b/examples/scalar-lump-mpi.py index 0185d8457..3498c63b3 100644 --- a/examples/scalar-lump-mpi.py +++ b/examples/scalar-lump-mpi.py @@ -221,7 +221,7 @@ def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: dv = eos.dependent_vars(state) if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) if resid is None: resid = state - exact viz_fields = [("cv", state), @@ -280,7 +280,7 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, state, exact) from mirgecom.simutil import allsync @@ -300,7 +300,7 @@ def my_pre_step(step, t, dt, state): if dv is None: dv = eos.dependent_vars(state) if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) resid = state - exact my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, resid=resid) @@ -308,7 +308,7 @@ def my_pre_step(step, t, dt, state): if do_status: if component_errors is None: if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, state, exact) my_write_status(component_errors) @@ -351,7 +351,7 @@ def my_rhs(t, state): logger.info("Checkpointing final state ...") final_dv = eos.dependent_vars(current_state) - final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) + final_exact = initializer(x_vec=nodes, eos=eos, time=current_t) final_resid = current_state - final_exact my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, exact=final_exact, resid=final_resid) diff --git a/examples/sod-mpi.py b/examples/sod-mpi.py index 0d7dbbef7..759c92941 100644 --- a/examples/sod-mpi.py +++ b/examples/sod-mpi.py @@ -211,7 +211,7 @@ def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: dv = eos.dependent_vars(state) if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) if resid is None: resid = state - exact viz_fields = [("cv", state), @@ -270,7 +270,7 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, state, exact) from mirgecom.simutil import allsync @@ -290,7 +290,7 @@ def my_pre_step(step, t, dt, state): if dv is None: dv = eos.dependent_vars(state) if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) resid = state - exact my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, resid=resid) @@ -298,7 +298,7 @@ def my_pre_step(step, t, dt, state): if do_status: if component_errors is None: if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = \ compare_fluid_solutions(discr, state, exact) @@ -342,7 +342,7 @@ def my_rhs(t, state): logger.info("Checkpointing final state ...") final_dv = eos.dependent_vars(current_state) - final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) + final_exact = initializer(x_vec=nodes, eos=eos, time=current_t) final_resid = current_state - final_exact my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, exact=final_exact, resid=final_resid) diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index b631c0e95..f7d37e96b 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -236,7 +236,7 @@ def my_write_viz(step, t, state, dv=None, exact=None, resid=None): if dv is None: dv = eos.dependent_vars(state) if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) if resid is None: resid = state - exact viz_fields = [("cv", state), @@ -294,7 +294,7 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, state, exact) from mirgecom.simutil import allsync @@ -313,7 +313,7 @@ def my_pre_step(step, t, dt, state): if do_status: if component_errors is None: if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) from mirgecom.simutil import compare_fluid_solutions component_errors = compare_fluid_solutions(discr, state, exact) my_write_status(state, component_errors) @@ -322,7 +322,7 @@ def my_pre_step(step, t, dt, state): if dv is None: dv = eos.dependent_vars(state) if exact is None: - exact = initializer(x_vec=nodes, eos=eos, t=t) + exact = initializer(x_vec=nodes, eos=eos, time=t) resid = state - exact my_write_viz(step=step, t=t, state=state, dv=dv, exact=exact, resid=resid) @@ -365,7 +365,7 @@ def my_rhs(t, state): logger.info("Checkpointing final state ...") final_dv = eos.dependent_vars(current_state) - final_exact = initializer(x_vec=nodes, eos=eos, t=current_t) + final_exact = initializer(x_vec=nodes, eos=eos, time=current_t) final_resid = current_state - final_exact my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv, exact=final_exact, resid=final_resid) From 3f0ea391c49b4301a51bdd34dc0ea98affa5be7e Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 20 Jul 2021 21:13:53 -0500 Subject: [PATCH 281/385] Merged AV, current Euler, NS --- .ci-support/production-testing-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index eb00d7217..1adeba830 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -15,7 +15,7 @@ set -x # patched by the incoming development. The following vars control the # production environment: # -export PRODUCTION_BRANCH="lift-euler-production" # The base production branch to be installed by emirge +# export PRODUCTION_BRANCH="" # The base production branch to be installed by emirge # export PRODUCTION_FORK="" # The fork/home of production changes (if any) # # Multiple production drivers are supported. The user should provide a ':'-delimited From dfcb4b5b9821a080533d00b14ead4b6eedd2410b Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 21 Jul 2021 11:26:54 -0500 Subject: [PATCH 282/385] Remove VTK from reqs --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f7e7c3458..b0654c66a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ pyvisfile pymetis importlib-resources psutil -vtk gmsh PyYAML From 59e7afbe07903a8e3365aaf6ea6622bf60437f88 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 21 Jul 2021 11:33:07 -0500 Subject: [PATCH 283/385] Remove jump operator, prefer built-in --- mirgecom/flux.py | 3 +-- mirgecom/operators.py | 22 ---------------------- 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/mirgecom/flux.py b/mirgecom/flux.py index e1db66ec5..fa760154e 100644 --- a/mirgecom/flux.py +++ b/mirgecom/flux.py @@ -34,7 +34,6 @@ """ import numpy as np # noqa from meshmode.dof_array import DOFArray -from mirgecom.operators import jump from mirgecom.fluid import ( ConservedVars, make_conserved @@ -162,4 +161,4 @@ def lfr_flux(cv_tpair, f_tpair, normal, lam): object array of :class:`meshmode.dof_array.DOFArray` with the Lax-Friedrichs/Rusanov flux. """ - return f_tpair.avg@normal - lam*jump(cv_tpair)/2 + return f_tpair.avg@normal - lam*cv_tpair.diff/2 diff --git a/mirgecom/operators.py b/mirgecom/operators.py index a61e868cf..b7db09898 100644 --- a/mirgecom/operators.py +++ b/mirgecom/operators.py @@ -4,7 +4,6 @@ .. autofunction:: dg_div .. autofunction:: element_boundary_flux .. autofunction:: elbnd_flux -.. autofunction:: jump """ __copyright__ = """ @@ -32,27 +31,6 @@ """ -# placeholder awaits resolution on grudge PR #71 -def jump(trace_pair): - r"""Return the "jump" in the quantities represented by the *trace_pair*. - - The jump in a quantity $\mathbf{q}$ is denoted $[\mathbf{q}]$ and is - defined by: - .. math: - [\mathbf{q}] = \mathbf{q}^+ - \mathbf{q}^- - - Parameters - ---------- - trace_pair: :class:`grudge.sym.TracePair` - Represents the quantity for which the jump is to be calculated. - - Returns - ------- - like(trace_pair.int) - """ - return trace_pair.ext - trace_pair.int - - def elbnd_flux(discr, compute_interior_flux, compute_boundary_flux, int_tpair, xrank_pairs, boundaries): """Generically compute flux across element boundaries.""" From ba3e59c5189d67f2339b29102bbd8293a021d8ee Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 21 Jul 2021 15:42:48 -0500 Subject: [PATCH 284/385] recover gmsh --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 681c5a6f6..a8fc69e34 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ pymetis importlib-resources psutil PyYAML +gmsh # The following packages will be git cloned by emirge: --editable git+https://github.com/inducer/pymbolic.git#egg=pymbolic From ddd960e02d091fa74abf9429fb310d32f2037901 Mon Sep 17 00:00:00 2001 From: jlevine18 Date: Thu, 22 Jul 2021 18:13:21 -0500 Subject: [PATCH 285/385] Add local maximum species diffusivity calculation (#420) Co-authored-by: Matt Smith Co-authored-by: Michael Campbell --- mirgecom/viscous.py | 48 ++++++++++++++++++++++++++++++++++++++++++-- test/test_viscous.py | 42 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/mirgecom/viscous.py b/mirgecom/viscous.py index 4a73c252b..433610730 100644 --- a/mirgecom/viscous.py +++ b/mirgecom/viscous.py @@ -42,13 +42,15 @@ """ import numpy as np +from pytools import memoize_in from pytools.obj_array import make_obj_array from mirgecom.fluid import ( velocity_gradient, species_mass_fraction_gradient, make_conserved ) -from meshmode.dof_array import thaw +from meshmode.dof_array import thaw, DOFArray +import arraycontext def viscous_stress_tensor(discr, eos, cv, grad_cv): @@ -245,12 +247,20 @@ def get_viscous_timestep(discr, eos, cv): length_scales = characteristic_lengthscales(cv.array_context, discr) mu = 0 + d_alpha_max = 0 transport = eos.transport_model() if transport: mu = transport.viscosity(eos, cv) + d_alpha = transport.species_diffusivity(eos, cv) + if d_alpha is not np.empty and d_alpha.size != 0: + d_alpha_max = \ + get_local_max_species_diffusivity( + cv.array_context, discr, d_alpha + ) return( - length_scales / (compute_wavespeed(eos, cv) + mu / length_scales) + length_scales / (compute_wavespeed(eos, cv) + + ((mu + d_alpha_max) / length_scales)) ) @@ -275,3 +285,37 @@ def get_viscous_cfl(discr, eos, dt, cv): The CFL at each node. """ return dt / get_viscous_timestep(discr, eos=eos, cv=cv) + + +def get_local_max_species_diffusivity(actx, discr, d_alpha): + """Return the maximum species diffusivity at every point. + + Parameters + ---------- + actx: :class:`arraycontext.ArrayContext` + Array context to use + discr: :class:`grudge.eager.EagerDGDiscretization` + the discretization to use + d_alpha: np.ndarray + Species diffusivities + """ + return_dof = [] + for i in range(len(d_alpha[0])): + stacked_diffusivity = actx.np.stack([x[i] for x in d_alpha]) + + n_species, ni1, ni0 = stacked_diffusivity.shape + + @memoize_in(discr, ("max_species_diffusivity", n_species, i)) + def make_max_kernel(): + # fun fact: arraycontext needs these exact loop names to work (even + # though a loopy kernel can have whatever iterator names the user wants) + # TODO: see if the opposite order [i0, i1, i2] is faster due to higher + # spatial locality, causing fewer cache misses + return arraycontext.make_loopy_program( + "{ [i1,i0,i2]: 0<=i1 Date: Thu, 22 Jul 2021 18:48:38 -0500 Subject: [PATCH 286/385] Fix up the diffusivity presence check --- mirgecom/viscous.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/viscous.py b/mirgecom/viscous.py index 433610730..6e2373c68 100644 --- a/mirgecom/viscous.py +++ b/mirgecom/viscous.py @@ -252,7 +252,7 @@ def get_viscous_timestep(discr, eos, cv): if transport: mu = transport.viscosity(eos, cv) d_alpha = transport.species_diffusivity(eos, cv) - if d_alpha is not np.empty and d_alpha.size != 0: + if len(d_alpha) > 0: d_alpha_max = \ get_local_max_species_diffusivity( cv.array_context, discr, d_alpha From 47f70d179ef04eeae4445e5c686fda14f68dc65d Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 22 Jul 2021 22:11:27 -0500 Subject: [PATCH 287/385] Merge main --- mirgecom/simutil.py | 2 ++ mirgecom/steppers.py | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 155121012..c9ef01c3a 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -155,6 +155,7 @@ def write_visfile(discr, io_fields, visualizer, vizname, comm = discr.mpi_communicator rank = 0 + if comm: rank = comm.Get_rank() @@ -165,6 +166,7 @@ def write_visfile(discr, io_fields, visualizer, vizname, viz_dir = os.path.dirname(rank_fn) if viz_dir and not os.path.exists(viz_dir): os.makedirs(viz_dir) + if comm: comm.barrier() diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 1490d6863..f7517d821 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -214,13 +214,11 @@ def _advance_state_leap(rhs, timestepper, state, if isinstance(state, np.ndarray): state = thaw(freeze(state, actx), actx) - if dt < 0: - return istep, t, state - if pre_step_callback is not None: state, dt = pre_step_callback(state=state, step=istep, t=t, dt=dt) + stepper_cls.dt = dt # Leap interface here is *a bit* different. stepper_cls.dt = dt @@ -233,6 +231,7 @@ def _advance_state_leap(rhs, timestepper, state, state, dt = post_step_callback(state=state, step=istep, t=t, dt=dt) + stepper_cls.dt = dt istep += 1 From 2ac137b8d0c6b11a0f392ec388795c1e20787da4 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 26 Jul 2021 07:06:05 -0500 Subject: [PATCH 288/385] Fixed missing raise. --- mirgecom/boundary.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 648b4702b..46a716d4d 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -112,6 +112,7 @@ def t_boundary_flux(self, discr, btag, cv, eos, **kwargs): def inviscid_boundary_flux(self, discr, btag, cv, eos, **kwargs): """Get the inviscid part of the physical flux across the boundary *btag*.""" + raise NotImplementedError() def viscous_boundary_flux(self, discr, btag, cv, grad_cv, grad_t, eos, **kwargs): """Get the viscous part of the physical flux across the boundary *btag*.""" @@ -119,6 +120,7 @@ def viscous_boundary_flux(self, discr, btag, cv, grad_cv, grad_t, eos, **kwargs) def boundary_pair(self, discr, btag, cv, eos, **kwargs): """Get the interior and exterior solution (*u*) on the boundary.""" + raise NotImplementedError() class PrescribedInviscidBoundary(FluidBC): From 5acba887cbe39300bf0b7d927fbaacc8fbea23f5 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 27 Jul 2021 14:22:21 -0500 Subject: [PATCH 289/385] Add exact soln to Poiseuille case --- examples/poiseuille-mpi.py | 93 ++++++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 28 deletions(-) diff --git a/examples/poiseuille-mpi.py b/examples/poiseuille-mpi.py index d33fe5b59..c93615d21 100644 --- a/examples/poiseuille-mpi.py +++ b/examples/poiseuille-mpi.py @@ -137,7 +137,8 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, raise ValueError("This example must be run with dim = 2.") left_boundary_location = 0 right_boundary_location = 0.1 - + ybottom = 0. + ytop = .02 rst_path = "restart_data/" rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" @@ -153,8 +154,8 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, assert restart_data["nparts"] == nparts else: # generate the grid from scratch npts_axis = (50, 30) - box_ll = (left_boundary_location, 0.0) - box_ur = (right_boundary_location, 0.02) + box_ll = (left_boundary_location, ybottom) + box_ur = (right_boundary_location, ytop) generate_mesh = partial(_get_box_mesh, 2, a=box_ll, b=box_ur, n=npts_axis) from mirgecom.simutil import generate_and_distribute_mesh local_mesh, global_nelements = generate_and_distribute_mesh(comm, @@ -189,32 +190,44 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, base_pressure = 100000.0 pressure_ratio = 1.001 + mu = 1.0 - def poiseuille_soln(nodes, eos, cv=None, **kwargs): - dim = len(nodes) + def poiseuille_2d(x_vec, eos, cv=None, **kwargs): + y = x_vec[1] + x = x_vec[0] x0 = left_boundary_location xmax = right_boundary_location xlen = xmax - x0 - p0 = base_pressure - p1 = pressure_ratio*p0 - p_x = p1 + p0*(1 - pressure_ratio)*(nodes[0] - x0)/xlen - ke = 0 - mass = nodes[0] + 1.0 - nodes[0] - momentum = make_obj_array([0*mass for i in range(dim)]) + p_low = base_pressure + p_hi = pressure_ratio*base_pressure + dp = p_hi - p_low + dpdx = dp/xlen + h = ytop - ybottom + u_x = dpdx*y*(h - y)/(2*mu) if exact else 0*x + p_x = p_hi - dpdx*x + rho = 1.0 + mass = 0*x + rho + u_y = 0*x + velocity = make_obj_array([u_x, u_y]) + ke = .5*np.dot(velocity, velocity)*mass + gamma = eos.gamma() if cv is not None: mass = cv.mass - momentum = cv.momentum - ke = .5*np.dot(cv.momentum, cv.momentum)/cv.mass - energy_bc = p_x / (eos.gamma() - 1) + ke - return make_conserved(dim, mass=mass, energy=energy_bc, - momentum=momentum) + vel = cv.velocity + ke = .5*np.dot(vel, vel)*mass + + rho_e = p_x/(gamma-1) + ke + return make_conserved(2, mass=mass, energy=rho_e, + momentum=mass*velocity) + + initializer = poiseuille_2d + eos = IdealSingleGas(transport_model=SimpleTransport(viscosity=mu)) + exact = initializer(x_vec=nodes, eos=eos) - initializer = poiseuille_soln boundaries = {DTAG_BOUNDARY("-1"): PrescribedViscousBoundary(q_func=initializer), DTAG_BOUNDARY("+1"): PrescribedViscousBoundary(q_func=initializer), DTAG_BOUNDARY("-2"): IsothermalNoSlipBoundary(), DTAG_BOUNDARY("+2"): IsothermalNoSlipBoundary()} - eos = IdealSingleGas(transport_model=SimpleTransport(viscosity=1.0)) if rst_filename: current_t = restart_data["t"] @@ -225,7 +238,7 @@ def poiseuille_soln(nodes, eos, cv=None, **kwargs): logmgr_set_time(logmgr, current_step, current_t) else: # Set the current state from time 0 - current_state = initializer(nodes=nodes, eos=eos) + current_state = exact vis_timer = None @@ -242,7 +255,7 @@ def poiseuille_soln(nodes, eos, cv=None, **kwargs): if rank == 0: logger.info(init_message) - def my_write_status(step, t, dt, dv, state): + def my_write_status(step, t, dt, dv, state, component_errors): from grudge.op import nodal_min, nodal_max p_min = nodal_min(discr, "vol", dv.pressure) p_max = nodal_max(discr, "vol", dv.pressure) @@ -257,13 +270,19 @@ def my_write_status(step, t, dt, dv, state): if rank == 0: logger.info(f"Step: {step}, T: {t}, DT: {dt}, CFL: {cfl}\n" f"----- Pressure({p_min}, {p_max})\n" - f"----- Temperature({t_min}, {t_max})\n") + f"----- Temperature({t_min}, {t_max})\n" + "----- errors=" + + ", ".join("%.3g" % en for en in component_errors)) - def my_write_viz(step, t, state, dv=None, exact=None): + def my_write_viz(step, t, state, dv=None): if dv is None: dv = eos.dependent_vars(state) + resid = state - exact viz_fields = [("cv", state), - ("dv", dv)] + ("dv", dv), + ("poiseuille", exact), + ("resid", resid)] + from mirgecom.simutil import write_visfile write_visfile(discr, viz_fields, visualizer, vizname=casename, step=step, t=t, overwrite=True) @@ -283,7 +302,7 @@ def my_write_restart(step, t, state): from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) - def my_health_check(state, dv): + def my_health_check(state, dv, component_errors): health_error = False from mirgecom.simutil import check_naninf_local, check_range_local if check_naninf_local(discr, "vol", dv.pressure): @@ -311,11 +330,18 @@ def my_health_check(state, dv): t_max = nodal_max(discr, "vol", dv.temperature) logger.info(f"Temperature range violation ({t_min=}, {t_max=})") + exittol = 10 + if max(component_errors) > exittol: + health_error = True + if rank == 0: + logger.info("Solution diverged from exact soln.") + return health_error def my_pre_step(step, t, dt, state): try: dv = None + component_errors = None if logmgr: logmgr.tick_before() @@ -328,9 +354,13 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) from mirgecom.simutil import allsync - health_errors = allsync(my_health_check(state, dv), comm, - op=MPI.LOR) + health_errors = allsync( + my_health_check(state, dv, component_errors), comm, + op=MPI.LOR + ) if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") @@ -350,7 +380,11 @@ def my_pre_step(step, t, dt, state): if do_status: # needed because logging fails to make output if dv is None: dv = eos.dependent_vars(state) - my_write_status(step=step, t=t, dt=dt, dv=dv, state=state) + if component_errors is None: + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + my_write_status(step=step, t=t, dt=dt, dv=dv, state=state, + component_errors=component_errors) except MyRuntimeError: if rank == 0: @@ -388,10 +422,13 @@ def my_rhs(t, state): final_dv = eos.dependent_vars(current_state) final_dt = get_sim_timestep(discr, current_state, current_t, current_dt, current_cfl, eos, t_final, constant_cfl) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, current_state, exact) + my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv) my_write_restart(step=current_step, t=current_t, state=current_state) my_write_status(step=current_step, t=current_t, dt=final_dt, dv=final_dv, - state=current_state) + state=current_state, component_errors=component_errors) if logmgr: logmgr.close() From cfc71499884c07e463b7ed0a85afbc3dcf6afdd4 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 27 Jul 2021 14:53:49 -0500 Subject: [PATCH 290/385] Correct initialization code in Poiseuille test. --- examples/poiseuille-mpi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/poiseuille-mpi.py b/examples/poiseuille-mpi.py index c93615d21..d083a31a0 100644 --- a/examples/poiseuille-mpi.py +++ b/examples/poiseuille-mpi.py @@ -203,7 +203,7 @@ def poiseuille_2d(x_vec, eos, cv=None, **kwargs): dp = p_hi - p_low dpdx = dp/xlen h = ytop - ybottom - u_x = dpdx*y*(h - y)/(2*mu) if exact else 0*x + u_x = dpdx*y*(h - y)/(2*mu) p_x = p_hi - dpdx*x rho = 1.0 mass = 0*x + rho From 4a2d5900195fb1564b3b477460867c81dfb5f404 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 28 Jul 2021 10:07:46 -0500 Subject: [PATCH 291/385] Evict gmsh from requirements.txt (#461) --- conda-env.yml | 1 + requirements.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-env.yml b/conda-env.yml index 1b17db5e5..da3c54ab1 100644 --- a/conda-env.yml +++ b/conda-env.yml @@ -21,3 +21,4 @@ dependencies: - cantera - h5py * nompi_* # Make sure cantera does not pull in MPI through h5py - vtk +- gmsh diff --git a/requirements.txt b/requirements.txt index a8fc69e34..681c5a6f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,6 @@ pymetis importlib-resources psutil PyYAML -gmsh # The following packages will be git cloned by emirge: --editable git+https://github.com/inducer/pymbolic.git#egg=pymbolic From 1ddeefe4c5ff2f875c2bdd950ccf0649f8c5d18b Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 29 Jul 2021 14:22:20 -0500 Subject: [PATCH 292/385] Add a blurb to limit species mass fraction. --- examples/autoignition-mpi.py | 2 ++ mirgecom/fluid.py | 5 +++++ mirgecom/simutil.py | 12 ++++++++++++ 3 files changed, 19 insertions(+) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 385bf9636..a34e9b955 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -437,6 +437,8 @@ def my_pre_step(step, t, dt, state): def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? # imo this is a design/scope flaw + from mirgecom.simutil import limit_species_mass_fractions + state = limit_species_mass_fractions(state) if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) diff --git a/mirgecom/fluid.py b/mirgecom/fluid.py index 193be1210..0b655eb32 100644 --- a/mirgecom/fluid.py +++ b/mirgecom/fluid.py @@ -243,6 +243,11 @@ def velocity(self): """Return the fluid velocity = momentum / mass.""" return self.momentum / self.mass + @property + def species_mass_fractions(self): + """Return the species mass fractions = species_mass / mass.""" + return self.species_mass / self.mass + def join(self): """Call :func:`join_conserved` on *self*.""" return join_conserved( diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index ad2f9318f..e4760419d 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -251,3 +251,15 @@ def create_parallel_grid(comm, generate_grid): "instead. This function will disappear August 1, 2021", DeprecationWarning, stacklevel=2) return generate_and_distribute_mesh(comm=comm, generate_mesh=generate_grid) + + +def limit_species_mass_fractions(cv): + """Keep the species mass fractions from going negative.""" + y = cv.species_mass_fractions + if len(y) > 0: + actx = cv.array_context + zero = 0 * y[0] + for y_spec in y: + y_spec = actx.np.where(y_spec < 0, zero, y_spec) + cv = cv.replace(species_mass=cv.mass*y) + return(cv) From dc9abbd6fa543883e4cd030ac15bb349f500eaec Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 3 Aug 2021 12:23:41 -0500 Subject: [PATCH 293/385] Switch temporarily to test --- .ci-support/production-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh index 00b7f228c..4f601929e 100644 --- a/.ci-support/production-install.sh +++ b/.ci-support/production-install.sh @@ -14,7 +14,7 @@ # patched by the incoming development. The following vars control the # production environment: # -# PRODUCTION_BRANCH = The production branch (default=y1-production) +PRODUCTION_BRANCH="y1-production-update" # PRODUCTION_FORK = The production fork (default=illinois-ceesd) # # If the environment file does not exist, the current development is From 0bc13548b5b50f6c93bc070c876a3ec50d8cbfac Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 3 Aug 2021 12:29:19 -0500 Subject: [PATCH 294/385] Undo production switch --- .ci-support/production-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh index 4f601929e..00b7f228c 100644 --- a/.ci-support/production-install.sh +++ b/.ci-support/production-install.sh @@ -14,7 +14,7 @@ # patched by the incoming development. The following vars control the # production environment: # -PRODUCTION_BRANCH="y1-production-update" +# PRODUCTION_BRANCH = The production branch (default=y1-production) # PRODUCTION_FORK = The production fork (default=illinois-ceesd) # # If the environment file does not exist, the current development is From 70c2ccf8f8a22aad70e9aa503a3d06817db1e8f3 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 3 Aug 2021 13:20:26 -0500 Subject: [PATCH 295/385] Fix up viscous timestep estimate for non-DOFArray diffusivities --- mirgecom/viscous.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mirgecom/viscous.py b/mirgecom/viscous.py index ff02e72d4..fd147d963 100644 --- a/mirgecom/viscous.py +++ b/mirgecom/viscous.py @@ -342,10 +342,13 @@ def get_viscous_timestep(discr, eos, cv): mu = transport.viscosity(eos, cv) d_alpha = transport.species_diffusivity(eos, cv) if len(d_alpha) > 0: - d_alpha_max = \ - get_local_max_species_diffusivity( - cv.array_context, discr, d_alpha - ) + if isinstance(d_alpha[0], DOFArray): + d_alpha_max = \ + get_local_max_species_diffusivity( + cv.array_context, discr, d_alpha + ) + else: + d_alpha_max = np.max(d_alpha) return( length_scales / (compute_wavespeed(eos, cv) From ba73afa4d8498c53fd88b21dd5030c4bac679871 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 3 Aug 2021 14:24:16 -0500 Subject: [PATCH 296/385] Fix up viscous timestep to deal with non DOFArrays --- mirgecom/viscous.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mirgecom/viscous.py b/mirgecom/viscous.py index fd147d963..e774a5d3c 100644 --- a/mirgecom/viscous.py +++ b/mirgecom/viscous.py @@ -342,13 +342,10 @@ def get_viscous_timestep(discr, eos, cv): mu = transport.viscosity(eos, cv) d_alpha = transport.species_diffusivity(eos, cv) if len(d_alpha) > 0: - if isinstance(d_alpha[0], DOFArray): - d_alpha_max = \ - get_local_max_species_diffusivity( - cv.array_context, discr, d_alpha - ) - else: - d_alpha_max = np.max(d_alpha) + d_alpha_max = \ + get_local_max_species_diffusivity( + cv.array_context, discr, d_alpha + ) return( length_scales / (compute_wavespeed(eos, cv) @@ -391,6 +388,9 @@ def get_local_max_species_diffusivity(actx, discr, d_alpha): d_alpha: np.ndarray Species diffusivities """ + if not isinstance(d_alpha[0], DOFArray): + return max(d_alpha) + return_dof = [] for i in range(len(d_alpha[0])): stacked_diffusivity = actx.np.stack([x[i] for x in d_alpha]) From f1c993870c1aa67b46bb99b375cce542304e35c7 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 3 Aug 2021 19:08:50 -0500 Subject: [PATCH 297/385] Fix per new transport API --- test/test_viscous.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_viscous.py b/test/test_viscous.py index 199a377a1..6cb92ad6f 100644 --- a/test/test_viscous.py +++ b/test/test_viscous.py @@ -107,7 +107,7 @@ def test_viscous_stress_tensor(actx_factory, transport_model): eos = IdealSingleGas(transport_model=tv_model) mu = tv_model.viscosity(eos, cv) - lam = tv_model.viscosity2(eos, cv) + lam = tv_model.volume_viscosity(eos, cv) # Exact answer for tau exp_grad_v = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) From 361cdec589e7f83da139a1a1eaf1e12b5a1e27e9 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 5 Aug 2021 14:50:11 -0500 Subject: [PATCH 298/385] Split inviscid/euler tests. --- test/test_euler.py | 365 +++-------------------------------------- test/test_inviscid.py | 369 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 391 insertions(+), 343 deletions(-) create mode 100644 test/test_inviscid.py diff --git a/test/test_euler.py b/test/test_euler.py index f1144b3c3..8b0ea0702 100644 --- a/test/test_euler.py +++ b/test/test_euler.py @@ -40,8 +40,6 @@ from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa -from grudge.eager import interior_trace_pair -from grudge.symbolic.primitives import TracePair from mirgecom.euler import euler_operator from mirgecom.fluid import make_conserved from mirgecom.initializers import Vortex2D, Lump, MulticomponentLump @@ -57,339 +55,22 @@ from grudge.shortcuts import make_visualizer -from mirgecom.inviscid import ( - get_inviscid_timestep, - inviscid_flux -) +from mirgecom.inviscid import get_inviscid_timestep from mirgecom.integrators import rk4_step logger = logging.getLogger(__name__) -@pytest.mark.parametrize("nspecies", [0, 1, 10]) -@pytest.mark.parametrize("dim", [1, 2, 3]) -def test_inviscid_flux(actx_factory, nspecies, dim): - """Identity test - directly check inviscid flux routine - :func:`mirgecom.inviscid.inviscid_flux` against the exact expected result. - This test is designed to fail if the flux routine is broken. - - The expected inviscid flux is: - F(q) = - """ - actx = actx_factory() - - nel_1d = 16 - - from meshmode.mesh.generation import generate_regular_rect_mesh - - # for dim in [1, 2, 3]: - mesh = generate_regular_rect_mesh( - a=(-0.5,) * dim, b=(0.5,) * dim, nelements_per_axis=(nel_1d,) * dim - ) - - order = 3 - discr = EagerDGDiscretization(actx, mesh, order=order) - eos = IdealSingleGas() - - logger.info(f"Number of {dim}d elems: {mesh.nelements}") - - def rand(): - from meshmode.dof_array import DOFArray - return DOFArray( - actx, - tuple(actx.from_numpy(np.random.rand(grp.nelements, grp.nunit_dofs)) - for grp in discr.discr_from_dd("vol").groups) - ) - - mass = rand() - energy = rand() - mom = make_obj_array([rand() for _ in range(dim)]) - - mass_fractions = make_obj_array([rand() for _ in range(nspecies)]) - species_mass = mass * mass_fractions - - cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom, - species_mass=species_mass) - - # {{{ create the expected result - - p = eos.pressure(cv) - escale = (energy + p) / mass - - numeq = dim + 2 + nspecies - - expected_flux = np.zeros((numeq, dim), dtype=object) - expected_flux[0] = mom - expected_flux[1] = mom * escale - - for i in range(dim): - for j in range(dim): - expected_flux[2+i, j] = (mom[i] * mom[j] / mass + (p if i == j else 0)) - - for i in range(nspecies): - expected_flux[dim+2+i] = mom * mass_fractions[i] - - expected_flux = make_conserved(dim, q=expected_flux) - - # }}} - - flux = inviscid_flux(discr, eos, cv) - flux_resid = flux - expected_flux - - for i in range(numeq, dim): - for j in range(dim): - assert (la.norm(flux_resid[i, j].get())) == 0.0 - - -@pytest.mark.parametrize("dim", [1, 2, 3]) -def test_inviscid_flux_components(actx_factory, dim): - """Test uniform pressure case. - - Checks that the Euler-internal inviscid flux routine - :func:`mirgecom.inviscid.inviscid_flux` returns exactly the expected result - with a constant pressure and no flow. - - Expected inviscid flux is: - F(q) = - - Checks that only diagonal terms of the momentum flux: - [ rho(V.x.V) + pI ] are non-zero and return the correctly calculated p. - """ - actx = actx_factory() - - eos = IdealSingleGas() - - p0 = 1.0 - - nel_1d = 4 - - from meshmode.mesh.generation import generate_regular_rect_mesh - mesh = generate_regular_rect_mesh( - a=(-0.5,) * dim, b=(0.5,) * dim, nelements_per_axis=(nel_1d,) * dim - ) - - order = 3 - discr = EagerDGDiscretization(actx, mesh, order=order) - eos = IdealSingleGas() - - logger.info(f"Number of {dim}d elems: {mesh.nelements}") - # === this next block tests 1,2,3 dimensions, - # with single and multiple nodes/states. The - # purpose of this block is to ensure that when - # all components of V = 0, the flux recovers - # the expected values (and p0 within tolerance) - # === with V = 0, fixed P = p0 - tolerance = 1e-15 - nodes = thaw(actx, discr.nodes()) - mass = discr.zeros(actx) + np.dot(nodes, nodes) + 1.0 - mom = make_obj_array([discr.zeros(actx) for _ in range(dim)]) - p_exact = discr.zeros(actx) + p0 - energy = p_exact / 0.4 + 0.5 * np.dot(mom, mom) / mass - cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) - p = eos.pressure(cv) - flux = inviscid_flux(discr, eos, cv) - assert discr.norm(p - p_exact, np.inf) < tolerance - logger.info(f"{dim}d flux = {flux}") - - # for velocity zero, these components should be == zero - assert discr.norm(flux.mass, 2) == 0.0 - assert discr.norm(flux.energy, 2) == 0.0 - - # The momentum diagonal should be p - # Off-diagonal should be identically 0 - assert discr.norm(flux.momentum - p0*np.identity(dim), np.inf) < tolerance - - -@pytest.mark.parametrize(("dim", "livedim"), [ - (1, 0), - (2, 0), - (2, 1), - (3, 0), - (3, 1), - (3, 2), - ]) -def test_inviscid_mom_flux_components(actx_factory, dim, livedim): - r"""Constant pressure, V != 0: - - Checks that the flux terms are returned in the proper order by running - only 1 non-zero velocity component at-a-time. - """ - actx = actx_factory() - - eos = IdealSingleGas() - - p0 = 1.0 - - nel_1d = 4 - - from meshmode.mesh.generation import generate_regular_rect_mesh - mesh = generate_regular_rect_mesh( - a=(-0.5,) * dim, b=(0.5,) * dim, nelements_per_axis=(nel_1d,) * dim - ) - - order = 3 - discr = EagerDGDiscretization(actx, mesh, order=order) - nodes = thaw(actx, discr.nodes()) - - tolerance = 1e-15 - for livedim in range(dim): - mass = discr.zeros(actx) + 1.0 + np.dot(nodes, nodes) - mom = make_obj_array([discr.zeros(actx) for _ in range(dim)]) - mom[livedim] = mass - p_exact = discr.zeros(actx) + p0 - energy = ( - p_exact / (eos.gamma() - 1.0) - + 0.5 * np.dot(mom, mom) / mass - ) - cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) - p = eos.pressure(cv) - assert discr.norm(p - p_exact, np.inf) < tolerance - flux = inviscid_flux(discr, eos, cv) - logger.info(f"{dim}d flux = {flux}") - vel_exact = mom / mass - - # first two components should be nonzero in livedim only - assert discr.norm(flux.mass - mom, np.inf) == 0 - eflux_exact = (energy + p_exact)*vel_exact - assert discr.norm(flux.energy - eflux_exact, np.inf) == 0 - - logger.info("Testing momentum") - xpmomflux = mass*np.outer(vel_exact, vel_exact) + p_exact*np.identity(dim) - assert discr.norm(flux.momentum - xpmomflux, np.inf) < tolerance - - -@pytest.mark.parametrize("nspecies", [0, 10]) -@pytest.mark.parametrize("order", [1, 2, 3]) -@pytest.mark.parametrize("dim", [1, 2, 3]) -def test_facial_flux(actx_factory, nspecies, order, dim): - """Check the flux across element faces by prescribing states (q) - with known fluxes. Only uniform states are tested currently - ensuring - that the Lax-Friedrichs flux terms which are proportional to jumps in - state data vanish. - - Since the returned fluxes use state data which has been interpolated - to-and-from the element faces, this test is grid-dependent. - """ - actx = actx_factory() - - tolerance = 1e-14 - p0 = 1.0 - - from meshmode.mesh.generation import generate_regular_rect_mesh - from pytools.convergence import EOCRecorder - - eoc_rec0 = EOCRecorder() - eoc_rec1 = EOCRecorder() - for nel_1d in [4, 8, 12]: - - mesh = generate_regular_rect_mesh( - a=(-0.5,) * dim, b=(0.5,) * dim, nelements_per_axis=(nel_1d,) * dim - ) - - logger.info(f"Number of elements: {mesh.nelements}") - - discr = EagerDGDiscretization(actx, mesh, order=order) - zeros = discr.zeros(actx) - ones = zeros + 1.0 - - mass_input = discr.zeros(actx) + 1.0 - energy_input = discr.zeros(actx) + 2.5 - mom_input = flat_obj_array( - [discr.zeros(actx) for i in range(discr.dim)] - ) - mass_frac_input = flat_obj_array( - [ones / ((i + 1) * 10) for i in range(nspecies)] - ) - species_mass_input = mass_input * mass_frac_input - - cv = make_conserved( - dim, mass=mass_input, energy=energy_input, momentum=mom_input, - species_mass=species_mass_input) - - from mirgecom.inviscid import inviscid_facial_flux - - # Check the boundary facial fluxes as called on an interior boundary - interior_face_flux = inviscid_facial_flux( - discr, eos=IdealSingleGas(), cv_tpair=interior_trace_pair(discr, cv)) - - def inf_norm(data): - if len(data) > 0: - return discr.norm(data, np.inf, dd="all_faces") - else: - return 0.0 - - assert inf_norm(interior_face_flux.mass) < tolerance - assert inf_norm(interior_face_flux.energy) < tolerance - assert inf_norm(interior_face_flux.species_mass) < tolerance - - # The expected pressure is 1.0 (by design). And the flux diagonal is - # [rhov_x*v_x + p] (etc) since we have zero velocities it's just p. - # - # The off-diagonals are zero. We get a {ndim}-vector for each - # dimension, the flux for the x-component of momentum (for example) is: - # f_momx = < 1.0, 0 , 0> , then we return f_momx .dot. normal, which - # can introduce negative values. - # - # (Explanation courtesy of Mike Campbell, - # https://github.com/illinois-ceesd/mirgecom/pull/44#discussion_r463304292) - - nhat = thaw(actx, discr.normal("int_faces")) - mom_flux_exact = discr.project("int_faces", "all_faces", p0*nhat) - print(f"{mom_flux_exact=}") - print(f"{interior_face_flux.momentum=}") - momerr = inf_norm(interior_face_flux.momentum - mom_flux_exact) - assert momerr < tolerance - eoc_rec0.add_data_point(1.0 / nel_1d, momerr) - - # Check the boundary facial fluxes as called on a domain boundary - dir_mass = discr.project("vol", BTAG_ALL, mass_input) - dir_e = discr.project("vol", BTAG_ALL, energy_input) - dir_mom = discr.project("vol", BTAG_ALL, mom_input) - dir_mf = discr.project("vol", BTAG_ALL, species_mass_input) - - dir_bc = make_conserved(dim, mass=dir_mass, energy=dir_e, - momentum=dir_mom, species_mass=dir_mf) - dir_bval = make_conserved(dim, mass=dir_mass, energy=dir_e, - momentum=dir_mom, species_mass=dir_mf) - boundary_flux = inviscid_facial_flux( - discr, eos=IdealSingleGas(), - cv_tpair=TracePair(BTAG_ALL, interior=dir_bval, exterior=dir_bc) - ) - - assert inf_norm(boundary_flux.mass) < tolerance - assert inf_norm(boundary_flux.energy) < tolerance - assert inf_norm(boundary_flux.species_mass) < tolerance - - nhat = thaw(actx, discr.normal(BTAG_ALL)) - mom_flux_exact = discr.project(BTAG_ALL, "all_faces", p0*nhat) - momerr = inf_norm(boundary_flux.momentum - mom_flux_exact) - assert momerr < tolerance - - eoc_rec1.add_data_point(1.0 / nel_1d, momerr) - - logger.info( - f"standalone Errors:\n{eoc_rec0}" - f"boundary Errors:\n{eoc_rec1}" - ) - assert ( - eoc_rec0.order_estimate() >= order - 0.5 - or eoc_rec0.max_error() < 1e-9 - ) - assert ( - eoc_rec1.order_estimate() >= order - 0.5 - or eoc_rec1.max_error() < 1e-9 - ) - - @pytest.mark.parametrize("nspecies", [0, 10]) @pytest.mark.parametrize("dim", [1, 2, 3]) @pytest.mark.parametrize("order", [1, 2, 3]) def test_uniform_rhs(actx_factory, nspecies, dim, order): - """Tests the inviscid rhs using a trivial constant/uniform state which - should yield rhs = 0 to FP. The test is performed for 1, 2, and 3 dimensions. - """ + """Test the inviscid rhs using a trivial constant/uniform state. + This state should yield rhs = 0 to FP. The test is performed for 1, 2, + and 3 dimensions, with orders 1, 2, and 3, with and without passive species. + """ actx = actx_factory() tolerance = 1e-9 @@ -512,11 +193,11 @@ def test_uniform_rhs(actx_factory, nspecies, dim, order): @pytest.mark.parametrize("order", [1, 2, 3]) def test_vortex_rhs(actx_factory, order): - """Tests the inviscid rhs using the non-trivial 2D isentropic vortex - case configured to yield rhs = 0. Checks several different orders and - refinement levels to check error behavior. - """ + """Test the inviscid rhs using the non-trivial 2D isentropic vortex. + The case is configured to yield rhs = 0. Checks several different orders + and refinement levels to check error behavior. + """ actx = actx_factory() dim = 2 @@ -567,9 +248,10 @@ def test_vortex_rhs(actx_factory, order): @pytest.mark.parametrize("dim", [1, 2, 3]) @pytest.mark.parametrize("order", [1, 2, 3]) def test_lump_rhs(actx_factory, dim, order): - """Tests the inviscid rhs using the non-trivial 1, 2, and 3D mass lump - case against the analytic expressions of the RHS. Checks several different - orders and refinement levels to check error behavior. + """Test the inviscid rhs using the non-trivial mass lump case. + + The case is tested against the analytic expressions of the RHS. + Checks several different orders and refinement levels to check error behavior. """ actx = actx_factory() @@ -630,9 +312,10 @@ def test_lump_rhs(actx_factory, dim, order): @pytest.mark.parametrize("order", [1, 2, 4]) @pytest.mark.parametrize("v0", [0.0, 1.0]) def test_multilump_rhs(actx_factory, dim, order, v0): - """Tests the inviscid rhs using the non-trivial 1, 2, and 3D mass lump case - against the analytic expressions of the RHS. Checks several different orders - and refinement levels to check error behavior. + """Test the Euler rhs using the non-trivial 1, 2, and 3D mass lump case. + + The case is tested against the analytic expressions of the RHS. Checks several + different orders and refinement levels to check error behavior. """ actx = actx_factory() nspecies = 10 @@ -701,11 +384,8 @@ def test_multilump_rhs(actx_factory, dim, order, v0): ) +# Basic timestepping loop for the Euler operator def _euler_flow_stepper(actx, parameters): - """ - Implements a generic time stepping loop for testing an inviscid flow - using a spectral filter. - """ logging.basicConfig(format="%(message)s", level=logging.INFO) mesh = parameters["mesh"] @@ -831,12 +511,11 @@ def rhs(t, q): @pytest.mark.parametrize("order", [2, 3, 4]) def test_isentropic_vortex(actx_factory, order): - """Advance the 2D isentropic vortex case in time with non-zero velocities - using an RK4 timestepping scheme. Check the advanced field values against - the exact/analytic expressions. + """Advance the 2D isentropic vortex case in time with non-zero velocities. - This tests all parts of the Euler module working together, with results - converging at the expected rates vs. the order. + This test uses an RK4 timestepping scheme, and checks the advanced field values + against the exact/analytic expressions. This tests all parts of the Euler module + working together, with results converging at the expected rates vs. the order. """ actx = actx_factory() diff --git a/test/test_inviscid.py b/test/test_inviscid.py new file mode 100644 index 000000000..37f0cea94 --- /dev/null +++ b/test/test_inviscid.py @@ -0,0 +1,369 @@ +"""Test the inviscid fluid module.""" + +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import numpy as np +import numpy.random +import numpy.linalg as la # noqa +import pyopencl.clmath # noqa +import logging +import pytest + +from pytools.obj_array import ( + flat_obj_array, + make_obj_array, +) + +from meshmode.dof_array import thaw +from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa +from grudge.eager import interior_trace_pair +from grudge.symbolic.primitives import TracePair +from mirgecom.fluid import make_conserved +from mirgecom.eos import IdealSingleGas +from grudge.eager import EagerDGDiscretization +from meshmode.array_context import ( # noqa + pytest_generate_tests_for_pyopencl_array_context + as pytest_generate_tests) +from mirgecom.inviscid import inviscid_flux + +logger = logging.getLogger(__name__) + + +@pytest.mark.parametrize("nspecies", [0, 1, 10]) +@pytest.mark.parametrize("dim", [1, 2, 3]) +def test_inviscid_flux(actx_factory, nspecies, dim): + """Check inviscid flux against exact expected result: Identity test. + + Directly check inviscid flux routine, :func:`mirgecom.inviscid.inviscid_flux`, + against the exact expected result. This test is designed to fail if the flux + routine is broken. + + The expected inviscid flux is: + F(q) = + """ + actx = actx_factory() + + nel_1d = 16 + + from meshmode.mesh.generation import generate_regular_rect_mesh + + # for dim in [1, 2, 3]: + mesh = generate_regular_rect_mesh( + a=(-0.5,) * dim, b=(0.5,) * dim, nelements_per_axis=(nel_1d,) * dim + ) + + order = 3 + discr = EagerDGDiscretization(actx, mesh, order=order) + eos = IdealSingleGas() + + logger.info(f"Number of {dim}d elems: {mesh.nelements}") + + def rand(): + from meshmode.dof_array import DOFArray + return DOFArray( + actx, + tuple(actx.from_numpy(np.random.rand(grp.nelements, grp.nunit_dofs)) + for grp in discr.discr_from_dd("vol").groups) + ) + + mass = rand() + energy = rand() + mom = make_obj_array([rand() for _ in range(dim)]) + + mass_fractions = make_obj_array([rand() for _ in range(nspecies)]) + species_mass = mass * mass_fractions + + cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom, + species_mass=species_mass) + + # {{{ create the expected result + + p = eos.pressure(cv) + escale = (energy + p) / mass + + numeq = dim + 2 + nspecies + + expected_flux = np.zeros((numeq, dim), dtype=object) + expected_flux[0] = mom + expected_flux[1] = mom * escale + + for i in range(dim): + for j in range(dim): + expected_flux[2+i, j] = (mom[i] * mom[j] / mass + (p if i == j else 0)) + + for i in range(nspecies): + expected_flux[dim+2+i] = mom * mass_fractions[i] + + expected_flux = make_conserved(dim, q=expected_flux) + + # }}} + + flux = inviscid_flux(discr, eos, cv) + flux_resid = flux - expected_flux + + for i in range(numeq, dim): + for j in range(dim): + assert (la.norm(flux_resid[i, j].get())) == 0.0 + + +@pytest.mark.parametrize("dim", [1, 2, 3]) +def test_inviscid_flux_components(actx_factory, dim): + """Test uniform pressure case. + + Checks that the Euler-internal inviscid flux routine + :func:`mirgecom.inviscid.inviscid_flux` returns exactly the expected result + with a constant pressure and no flow. + + Expected inviscid flux is: + F(q) = + + Checks that only diagonal terms of the momentum flux: + [ rho(V.x.V) + pI ] are non-zero and return the correctly calculated p. + """ + actx = actx_factory() + + eos = IdealSingleGas() + + p0 = 1.0 + + nel_1d = 4 + + from meshmode.mesh.generation import generate_regular_rect_mesh + mesh = generate_regular_rect_mesh( + a=(-0.5,) * dim, b=(0.5,) * dim, nelements_per_axis=(nel_1d,) * dim + ) + + order = 3 + discr = EagerDGDiscretization(actx, mesh, order=order) + eos = IdealSingleGas() + + logger.info(f"Number of {dim}d elems: {mesh.nelements}") + # === this next block tests 1,2,3 dimensions, + # with single and multiple nodes/states. The + # purpose of this block is to ensure that when + # all components of V = 0, the flux recovers + # the expected values (and p0 within tolerance) + # === with V = 0, fixed P = p0 + tolerance = 1e-15 + nodes = thaw(actx, discr.nodes()) + mass = discr.zeros(actx) + np.dot(nodes, nodes) + 1.0 + mom = make_obj_array([discr.zeros(actx) for _ in range(dim)]) + p_exact = discr.zeros(actx) + p0 + energy = p_exact / 0.4 + 0.5 * np.dot(mom, mom) / mass + cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) + p = eos.pressure(cv) + flux = inviscid_flux(discr, eos, cv) + assert discr.norm(p - p_exact, np.inf) < tolerance + logger.info(f"{dim}d flux = {flux}") + + # for velocity zero, these components should be == zero + assert discr.norm(flux.mass, 2) == 0.0 + assert discr.norm(flux.energy, 2) == 0.0 + + # The momentum diagonal should be p + # Off-diagonal should be identically 0 + assert discr.norm(flux.momentum - p0*np.identity(dim), np.inf) < tolerance + + +@pytest.mark.parametrize(("dim", "livedim"), [ + (1, 0), + (2, 0), + (2, 1), + (3, 0), + (3, 1), + (3, 2), + ]) +def test_inviscid_mom_flux_components(actx_factory, dim, livedim): + r"""Test components of the momentum flux with constant pressure, V != 0. + + Checks that the flux terms are returned in the proper order by running + only 1 non-zero velocity component at-a-time. + """ + actx = actx_factory() + + eos = IdealSingleGas() + + p0 = 1.0 + + nel_1d = 4 + + from meshmode.mesh.generation import generate_regular_rect_mesh + mesh = generate_regular_rect_mesh( + a=(-0.5,) * dim, b=(0.5,) * dim, nelements_per_axis=(nel_1d,) * dim + ) + + order = 3 + discr = EagerDGDiscretization(actx, mesh, order=order) + nodes = thaw(actx, discr.nodes()) + + tolerance = 1e-15 + for livedim in range(dim): + mass = discr.zeros(actx) + 1.0 + np.dot(nodes, nodes) + mom = make_obj_array([discr.zeros(actx) for _ in range(dim)]) + mom[livedim] = mass + p_exact = discr.zeros(actx) + p0 + energy = ( + p_exact / (eos.gamma() - 1.0) + + 0.5 * np.dot(mom, mom) / mass + ) + cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) + p = eos.pressure(cv) + assert discr.norm(p - p_exact, np.inf) < tolerance + flux = inviscid_flux(discr, eos, cv) + logger.info(f"{dim}d flux = {flux}") + vel_exact = mom / mass + + # first two components should be nonzero in livedim only + assert discr.norm(flux.mass - mom, np.inf) == 0 + eflux_exact = (energy + p_exact)*vel_exact + assert discr.norm(flux.energy - eflux_exact, np.inf) == 0 + + logger.info("Testing momentum") + xpmomflux = mass*np.outer(vel_exact, vel_exact) + p_exact*np.identity(dim) + assert discr.norm(flux.momentum - xpmomflux, np.inf) < tolerance + + +@pytest.mark.parametrize("nspecies", [0, 10]) +@pytest.mark.parametrize("order", [1, 2, 3]) +@pytest.mark.parametrize("dim", [1, 2, 3]) +def test_facial_flux(actx_factory, nspecies, order, dim): + """Check the flux across element faces. + + The flux is checked by prescribing states (q) with known fluxes. Only uniform + states are tested currently - ensuring that the Lax-Friedrichs flux terms which + are proportional to jumps in state data vanish. + + Since the returned fluxes use state data which has been interpolated + to-and-from the element faces, this test is grid-dependent. + """ + actx = actx_factory() + + tolerance = 1e-14 + p0 = 1.0 + + from meshmode.mesh.generation import generate_regular_rect_mesh + from pytools.convergence import EOCRecorder + + eoc_rec0 = EOCRecorder() + eoc_rec1 = EOCRecorder() + for nel_1d in [4, 8, 12]: + + mesh = generate_regular_rect_mesh( + a=(-0.5,) * dim, b=(0.5,) * dim, nelements_per_axis=(nel_1d,) * dim + ) + + logger.info(f"Number of elements: {mesh.nelements}") + + discr = EagerDGDiscretization(actx, mesh, order=order) + zeros = discr.zeros(actx) + ones = zeros + 1.0 + + mass_input = discr.zeros(actx) + 1.0 + energy_input = discr.zeros(actx) + 2.5 + mom_input = flat_obj_array( + [discr.zeros(actx) for i in range(discr.dim)] + ) + mass_frac_input = flat_obj_array( + [ones / ((i + 1) * 10) for i in range(nspecies)] + ) + species_mass_input = mass_input * mass_frac_input + + cv = make_conserved( + dim, mass=mass_input, energy=energy_input, momentum=mom_input, + species_mass=species_mass_input) + + from mirgecom.inviscid import inviscid_facial_flux + + # Check the boundary facial fluxes as called on an interior boundary + interior_face_flux = inviscid_facial_flux( + discr, eos=IdealSingleGas(), cv_tpair=interior_trace_pair(discr, cv)) + + def inf_norm(data): + if len(data) > 0: + return discr.norm(data, np.inf, dd="all_faces") + else: + return 0.0 + + assert inf_norm(interior_face_flux.mass) < tolerance + assert inf_norm(interior_face_flux.energy) < tolerance + assert inf_norm(interior_face_flux.species_mass) < tolerance + + # The expected pressure is 1.0 (by design). And the flux diagonal is + # [rhov_x*v_x + p] (etc) since we have zero velocities it's just p. + # + # The off-diagonals are zero. We get a {ndim}-vector for each + # dimension, the flux for the x-component of momentum (for example) is: + # f_momx = < 1.0, 0 , 0> , then we return f_momx .dot. normal, which + # can introduce negative values. + # + # (Explanation courtesy of Mike Campbell, + # https://github.com/illinois-ceesd/mirgecom/pull/44#discussion_r463304292) + + nhat = thaw(actx, discr.normal("int_faces")) + mom_flux_exact = discr.project("int_faces", "all_faces", p0*nhat) + print(f"{mom_flux_exact=}") + print(f"{interior_face_flux.momentum=}") + momerr = inf_norm(interior_face_flux.momentum - mom_flux_exact) + assert momerr < tolerance + eoc_rec0.add_data_point(1.0 / nel_1d, momerr) + + # Check the boundary facial fluxes as called on a domain boundary + dir_mass = discr.project("vol", BTAG_ALL, mass_input) + dir_e = discr.project("vol", BTAG_ALL, energy_input) + dir_mom = discr.project("vol", BTAG_ALL, mom_input) + dir_mf = discr.project("vol", BTAG_ALL, species_mass_input) + + dir_bc = make_conserved(dim, mass=dir_mass, energy=dir_e, + momentum=dir_mom, species_mass=dir_mf) + dir_bval = make_conserved(dim, mass=dir_mass, energy=dir_e, + momentum=dir_mom, species_mass=dir_mf) + boundary_flux = inviscid_facial_flux( + discr, eos=IdealSingleGas(), + cv_tpair=TracePair(BTAG_ALL, interior=dir_bval, exterior=dir_bc) + ) + + assert inf_norm(boundary_flux.mass) < tolerance + assert inf_norm(boundary_flux.energy) < tolerance + assert inf_norm(boundary_flux.species_mass) < tolerance + + nhat = thaw(actx, discr.normal(BTAG_ALL)) + mom_flux_exact = discr.project(BTAG_ALL, "all_faces", p0*nhat) + momerr = inf_norm(boundary_flux.momentum - mom_flux_exact) + assert momerr < tolerance + + eoc_rec1.add_data_point(1.0 / nel_1d, momerr) + + logger.info( + f"standalone Errors:\n{eoc_rec0}" + f"boundary Errors:\n{eoc_rec1}" + ) + assert ( + eoc_rec0.order_estimate() >= order - 0.5 + or eoc_rec0.max_error() < 1e-9 + ) + assert ( + eoc_rec1.order_estimate() >= order - 0.5 + or eoc_rec1.max_error() < 1e-9 + ) From f2c97807078344020711815f666331b71a90faa8 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 13 Aug 2021 10:58:17 -0500 Subject: [PATCH 299/385] Fix misplaced rank check --- examples/nsmix-mpi.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/nsmix-mpi.py b/examples/nsmix-mpi.py index a83e0cacb..5bdc8fb97 100644 --- a/examples/nsmix-mpi.py +++ b/examples/nsmix-mpi.py @@ -303,14 +303,14 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, f" {eq_density=}, {eq_mass_fractions=}") def my_write_status(step, t, dt, state): + if constant_cfl: + cfl = current_cfl + else: + from mirgecom.viscous import get_viscous_cfl + cfl_field = get_viscous_cfl(discr, eos, dt, cv=state) + from grudge.op import nodal_max + cfl = nodal_max(discr, "vol", cfl_field) if rank == 0: - if constant_cfl: - cfl = current_cfl - else: - from mirgecom.viscous import get_viscous_cfl - cfl_field = get_viscous_cfl(discr, eos, dt, cv=state) - from grudge.op import nodal_max - cfl = nodal_max(discr, "vol", cfl_field) logger.info(f"Step: {step}, T: {t}, DT: {dt}, CFL: {cfl}") def my_write_viz(step, t, state, dv=None, production_rates=None): From 8830bb1ba879c1977d29406514fd612501c0d187 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 21 Aug 2021 08:40:52 -0500 Subject: [PATCH 300/385] Update from upstream --- test/test_av.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_av.py b/test/test_av.py index c3465e281..858afd7b4 100644 --- a/test/test_av.py +++ b/test/test_av.py @@ -194,8 +194,8 @@ def av_flux(self, disc, btag, diffusion, **kwargs): from grudge.trace_pair import TracePair bnd_grad_pair = TracePair(btag, interior=grad_soln_minus, exterior=grad_soln_plus) - from mirgecom.flux import gradient_flux_central - flux_weak = gradient_flux_central(bnd_grad_pair, normal=nhat) + from mirgecom.flux import divergence_flux_central + flux_weak = divergence_flux_central(bnd_grad_pair, normal=nhat) return disc.project(btag, "all_faces", flux_weak) boundaries = {BTAG_ALL: TestBoundary()} From 2cdf5b9e859b301cde2f36a130045b576463c9b7 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sat, 21 Aug 2021 09:52:08 -0500 Subject: [PATCH 301/385] Add back gmsh to requirements --- examples/doublemach-mpi.py | 3 ++- requirements.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 42a22a714..206973fe8 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -116,7 +116,8 @@ def get_doublemach_mesh(): Physical Curve('ic3') = {1}; Physical Curve('wall') = {2}; Physical Curve('out') = {5}; - """, "geo"), force_ambient_dim=2, dimensions=2, target_unit="M") + """, "geo"), force_ambient_dim=2, dimensions=2, target_unit="M", + output_file_name=meshfile) else: mesh = read_gmsh(meshfile, force_ambient_dim=2) diff --git a/requirements.txt b/requirements.txt index afefc2614..326fe7806 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ pymetis importlib-resources psutil pyyaml +gmsh # The following packages will be git cloned by emirge: --editable git+https://github.com/inducer/pymbolic.git#egg=pymbolic From 9caa2ec67d4bdd84203f485114672f6de2bb9190 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Sat, 21 Aug 2021 10:47:05 -0500 Subject: [PATCH 302/385] Y1 production update (#487) Co-authored-by: Matt Smith --- .gitignore | 3 - doc/figures/poiseuille.png | Bin 0 -> 67499 bytes doc/support/operators.rst | 4 + doc/support/support.rst | 1 + examples/doublemach-mpi.py | 3 +- examples/poiseuille-mpi.py | 2 +- mirgecom/boundary.py | 16 +- mirgecom/eos.py | 47 +++++- mirgecom/fluid.py | 5 + mirgecom/flux.py | 56 +++---- mirgecom/initializers.py | 118 ++++++++++++- mirgecom/inviscid.py | 4 +- mirgecom/navierstokes.py | 4 +- mirgecom/operators.py | 4 +- mirgecom/simutil.py | 5 +- mirgecom/steppers.py | 2 +- mirgecom/viscous.py | 89 +++++----- test/test_av.py | 8 +- test/test_bc.py | 8 +- test/test_navierstokes.py | 330 +++++++++++++++++++++++++++++++++++++ test/test_operators.py | 287 ++++++++++++++++++++++++++++++++ test/test_viscous.py | 204 ++++++++++++++++++++--- 22 files changed, 1066 insertions(+), 134 deletions(-) create mode 100644 doc/figures/poiseuille.png create mode 100644 doc/support/operators.rst create mode 100644 test/test_navierstokes.py create mode 100644 test/test_operators.py diff --git a/.gitignore b/.gitignore index 63711c463..5c0c82d53 100644 --- a/.gitignore +++ b/.gitignore @@ -41,9 +41,6 @@ test/nodal-dg *.sqlite *.sqlite-journal -# Emacs backups -.#* -\#* # Linting temp files .run-pylint.py diff --git a/doc/figures/poiseuille.png b/doc/figures/poiseuille.png new file mode 100644 index 0000000000000000000000000000000000000000..39db2708188cdd35c41bf3b00c55991f7395bdc9 GIT binary patch literal 67499 zcmd43c|6o@_&=%zMR;0}y+X1VBiYMVcG-6=WGDM>Dy34$GKB0Q%NXlm29qQqTb98X zl&piXjctrM_w+p9bAG?`dYwPcU*~wnGt>Bd=JUDl`?}uO`+8s3J+F-oZ?iCQGttq} zvFO~aNgm1eQ8)G*EZj+3zvS! zzVg+WIV$!lqv80e<2#bq-h{rj*8kxr(9_fG+ZlIlk89_OYrb#$r-2js`B(Cfa2~&Z z^9VifwsTQh={{w^Ue3JPpzr9ZHHvbdvN3)*5>;AOwxlvYB-$2eZwYROjxM(+z5M9k zS0})Tz|TLZBY&Usc>X@A%m00P?)~@a<^pZNbWi$O{{Hj7WBxnkLclTFyZ;?S_unc1 zUwkb3&51H_m-xPKaXfX8dbpve+X?(j<8>Zd2sj)L^TH*4%*fzgslz&y8`II*yh;V5 z3(}~^2W4*KeAKAKE^$QpjmK>K%A-*H?)r>%`oGh1v;XdA= zjs#-{6Y)f4vOiBl&hO71C@L@S7In8nErT6ZF!1TJLB`jvahXeBcKIDTI@)Yqm;#qEv$tO{>ieKzDsEAJMS*&ZP5GaUmbEX; z*02A&mup#yfj=APvk68@@dYpW!#*uEbvl_OC~v${!`HYNWku~TmTUg|fw?Jfq)=BN zDuGKw`I`or*9~~Cd#)9W7-W2Og>*6fcWb%|iyxCAuKgFFVTT8MVZ`PgrnzpSO}4-Y zum{>g|G9RmV0wJ{;K*>x?r&z5oeBTLt;+jtkci#K7W{O7m%E>Kp*u16cL&*2w%+k} ztQQMJ?cR5mN$)@S&uGN1f-7W&{C<7770WIT@t@O&Dvs7EpLxBIIdyPZ9GDf&;69lh zs-S77>iqojf>|zj)sDLEJM*o?65&vO^}jKh%x8yJC?FY88(ryp`>}=X~T}OYMP8wIB2<5r=qu zntmA>Wk7VQtlqNI)7|8evBL&hgHK!Liecp|)q2Ch9_hV0(a3%6!Ey1EbWg5p>VaDg ze|IqA-Pg(}pg+yX;xL znrk1lGiepvwKxU-enb*nV)>Rs*rKr7W4)jBoOU>Rrg(4cyAaJun0>pFRz8Y^_h;d4M*rk zdtDd7r^(lWE3P!}Olsv2CK_udq0~qcaViH))aRhDa(h|4@nP z$qDc#U4S$rIN8}cRjTpNnI2%zb#--$2WxG#TZ$7GF zmr~kkG|23q=+A0SRVuM1Mwa|(s-Gk$adRi--5UNEW$02)PDF|C4A#6s z`h};$?TTM!&yD|&mHqAq15N5oiGmWhE<0|L!;8h!^#{iPbBGk?iKK%gL;O+uJ=O4u zY-e%oqR-&u2BR5s3nu^_0Nzh-t_AVl?sVGoXxzm6^($Q5stYG!{6a$Rsd$Wd1OhwEs+tslFXTgNIa-nFp zpDBAvqsXnRT0-m6X{}2y0#_$zQk^>?21l>c(Rn?mq4)NaO1?O4_BZV*cZ5<5i<#%* zI%af*&cwDqh}ABjS<7EHG#lMMB-zwAf6nT3ij7&-y!4CO{@@yB7UPz7UaF zwD{7y!#AbVDMh!CX^v!riq6ZVUF7d)Xile|TXOKmA@c`5F9!$t^brie>?^vMO_K{yoRD|#*SV-D&-K;j4Q${WRqyi* zQvgJ0FoMshTtQXLN!tp9d5`8+&@2x8`zI{EY7#$?M#Zygk1Beja$_Mc`>1DB)? zTV6H_cRZHllI-drJOIK-|b&Ru3LHg!`_aI*!_6xAqzc#H?XacNR=2Y|u zFh;uN>-RJD#E+=&u;La&@|4o`Ih1qsFFT))k?3X$lw|}*1hxA+O2Lnh*W*`H?@zLJ zAXk#|fWIoln^rTkg1=XaXHmzI0K-hz>@Ft*-1@L{^!X|h_X`r-(Z+fL9D!bH^r=j9 z5rc`n7lG-DCU@?#7=9pwgRbNE&hgxl9PR2cq+JQU-{@nSX2GB#Pp2N7`I4wB@b~(| z{XK=So8&s&q>RL_PhsB?y75V*7p^ucG6;ePfml58PNhk#6*>sc&CTtUL5m>WOh@|5 z{U_(_Y&!3`gIM^P-)XWb6t!5Cd9WCwT%mTjo={p*;XOib+U!+Akw$DOi#2_!wtx(* zO4aa3ZDvlnwmzdz`lxv+4^3=Q+B#UNpQ;$EanVcC9~^Q#+~71fH(xo^9ke~>LU_UH z@J>i&t6w8*u`q3@zkgZ{3SKOIOg401Cj(X`UeR!AD;q;u@u}OH3@7%VopQ+z)%Kt7 z9qtf-miMVq?Nq|H`kx#*wl$21GI!kTvdmEKq>N)*imYoM=_F-GZ0JO;csM>}Jd&GF zVN0_(3Y_7J+xqYJ48=hAg(^Tn@j*DRp0FIiEnZ9(`%JZ<%@(4TN{S&ay}W8>^DkF9 z9YSsauDMz4(>R-Ie<@%JN~J_it1je5L+*b$PCGk_{+W>1WK*Mg9=R~h;U}PFGe*J7 zqj#Zu-Ht({GQIDwXbbgv! z+83TRQofs|6#Pi!NUrrS^v{kX%Qr5U(cKL9C$H3d%>76&e`P;WYF-x5BZqzdrP$dT zwe?d^W0S@OZR)-4-*oR09I4zH**ARi-QC@TzRRd?{*YCiO)=GF@a|J0HWkdoD;eg1 zY3U5rPT+^j_TJD)+8T4oSseF3I13&%_MK)uv|aS40qyp5@=HCbMors#Z)fM_FD;u{ z#9gdk0++1z!*6dBg|dPtL-UM37#5Sh3ntWldfSsJPk1eoeL^K@%sFl4-E26x;Xp9x z7byQAFdGj$;)iMD6O6y1p6a~UC+W~+)xjTLo+@c;v7zFFTD(k^3!aLsr*Ww3LVuCI zf7{)|gOr?e@SHtS7LXN#Y-Q_Y5u*`6uX0)W6l)26@uL^JLuj+sr3z>q>DNnvCBWSWa}Kvent>6A zQApTfxUD=i8zyeKG?(ckwUNKp4ZifZZeZT*5fB&$Bz;55 zLB;GI5#8LhJ?;@k8n&b?0#av-XxT5_-ysTQ5W##7{YaM`F4W_sRx$F2+{}~*cQc7~ zgf>r&+uPcfy%dOgF93Z0`RX=T6H8nFXYbP_;)7Bsr|B`Y)Q8_z~_Va9?uqor?H9x_Glqc1RA0c44p)K-(!%q`ufAP=qS6t(?jx9dN65+A z!j+$HrMrc^TGsxClRT)GCN-P@J$Oi2G=jQs-o!X{o{_YwyvfWV(Z4aN}C3lcA5KBXYf;dyvj*(M_4x`Aml zBO9v*(J)JKF9!osKL5EN17MOXz!JJoe||a+3@Wfov=~B0dcv4uG>P2v z52Tv6OehPt)i#QY4YNvfs0Z`xX`A3u&lwmLffEhad0_dfvl!Z1oUJo%_&ylhlLoAr zvXO?_Dke6}u-XR>n+wkis6{n_S0c=%8p$ew_K?~R|qgX5`_qL1gIvdK+9vw!->uH zorGvhxJ_Zs!5@*ZKZ&>1cV74I%t}M8fJ6+T4<+1Fr>7=J?j*v`(tZGlBF_+qulxpWt? z#5?D*c0kUA?Tgx$EgW>y`T2~Vc?+auu3ulBW;f4J1|E~wvB78zVpnDqsm!s{_k)Er z7Wnh5Wq^cZi(O9Sb~<(9^ljceuoke$NK6z30*of|V7(*hBS3DW%pgv56?i0;`r^O?zM~h7nMTNApIPc-Tg# zh%NpMv5`=LR?l%h*j*Q%U%{j6dvDCV?O&KNEjEN-3Bc?xmD!*IO4OQ(E##%=XAMvE z)ZA~2xlYJXaUZ&(7J%8C%gmdPVH0o|26y1xxF;32jW=MqaW`t ziiS&XjE~!_fGJwOK$oJ-xGe>*^@qh-tVX2yJKUQeU#VE!Ru2|Bb61()(%c-T7_dMC z!B3^7y6Yi!iFcm&v(!EWfnx;VGFO=AShcexQ`s#Lc)SKaH}|QMU4Oj4w*ND&8ihp0J-O>tcB2 zG28j8H>VEf+-B)-TN0iXlm#xX+c(snxe(x!lNHa2 zdRX-yq%E{OW@*VCP=!7F9|rD+Pl8rz2Mq^>)9lWKEypXZy85n#fYo7Uz-+9hHv!O4 zp&o3O!r{Y7!g@?L0>Bau`{kPc4Q+sAtQC{D77G9g@t`Ryzh8@N0g!d={h*Mc6zu&e z@F#MmI@-0aE{C%0VuSQdFVauX2ZGL>?o`RaotB;EXVa7&47GXKd3bo(4Sd4k*7rm~ zy&%b*w|q(#lzq}>u^PDiWy}o7{uY62k%|EfNTC+;pY1(MJ5y&-S=kK=f^CSmVq?t zU@=|RB_^8+jNY=gZ0*%)!L2{ZMjX#~`RWd7jP3b$FhE=+&Re{JaUXy{jYXO~i1NnR zje}JD1Bq_26F&?BsZij~8o1Ak#W@wKk^ARd8zBZI#;9>hjwA)~ju>#Ct z!`RgHY=*bH+2N6649{DX5M>HWL!(a-aVLcggd5M%G7tW&^X@*QpnO<3W{tRk2cy!t3TRfj_ zzi#AwR`3!CSmnDa8|WD7G^3hvU$#m&&5cx106{boB0jt9Rv4=*GYia|Dftl4=3n}jXkHtZP{wK&prYqZ8r zAzrp_O})f1E`0T;N4G?doa0KZMgJ~qR-H_-v;*np(4Z0>A9b}Ab;}y zB7dJ}G!P#~fYbk)sqU5%UzoFXTj<4DAX5(ZIHc?rs8rxJr|QnR$Z%0;tTk+Vx#X1Cr~xET66v(Y&W4G~cL( zG2=82Z~C;_nx-a02}oU3;rtHAcgGI8>8wpZH{t#$Eh~DuaWsf1ixucqraD&;MvM2s zl3Vptn%;xRt(LcVE1JRF%=_@ukU->CA{E+Q9kAIWPq3ZZw^*R&FfbO_Z#G;PF2 z9d7270r8Ss-uuH}v*1%gF2Fm+Z2-c2vh^!`>P=c2R+0?J@$Xew`dRp9%5)l#xU{9E zCH0+4Xca$;5rk7GTQ>|DFyDtAYHof5*8mai0mwJ8 z9;9Aq_N^hn7K2PnU%cjHtt4>CK6)#KUY5Z}?%)p}(x`6BAqgQrR+=e*+PIx4RS2aX z%A9JON;^(umUOmt*%|kcHh6#amMzj#bf4d2qx90^ld{U_2QRwHr*JFB7)X zU9ty&J3BxE0RC@_T7^8 z6DIwN;d`@b@?$;9RGV>KTUFB%GgrscyjDH_<8H-nzrMUct$*W@i`ZSOCf2(Ti_x60 ze!{OiXo}I|67&K>m+^R6fJz$or;3yV3&MfaTo2bxuka;2I@nbaYTRt1O4%Ch4-l5b z!|jXdwwu;K4ojC8{*oSOLQXhjl!~*%LLWNvR4dOYDFpvx^H`r8*3NW0u$An@$*fiw z0bnC<(Zn7}n?eeh2rS4$%Z$`0W`N)&09K6JtHiX1cygl%r+kjFI|CZ>YfdSk1f)9n0S({x*f_yYN<2-9GEVA?8( zZN^$tm_w4Zy>B}^&Y90^yjF)t-rKX9>=!kw;h__wG;4L%gwo-!P^3nA3=~;S^s*L9wBdF+QIAp zHeO-wJdFhHw@XL;Qrr2$EbUp}FR4R4+`$9_f}5H(-zk!F%0M7^!WS+ktM-q@_-}rQ z+fO0;@Rhp5+=mHjhYbKM6e%+T)GH_E4XzoOiev?tWFm0cD&g*H)2%e2U5J)&4Q9xB zbg??@02VAVC~1x%l+3bF5$#K-=NK=UnjHjN&8Z32h6 zoyj5|8-F@c0JPpb;`j#@{2D0KL7n}46MJ-{a#?;|q9{Og#WR3lQI*%9X;>UM3$sk+ zec}ZYE81T)(mri14&J7*)Q_Hex2~Q?Rg@8BVp_aE2)Jy8D;Lrm&yB*{gXlHf93C8XKo^@01T#L=i$am)PJqU} z;kZPQf;j*#J`5@zB#_B@?!<=tgTh6;9Gu`|Q@U^u8#rhn?*`IA_8xMF+TmTGJ`75? z?B;8~J)Z|y(SBO~M-$N=pwb9xzwLj^C996WfU&6WsInjfbjPF!K-ltLaF{gNPQ9Ukyc&hB6< zVIgw{Xj)7xQJe=-ilEX|xf&b;k_0!fG=~A8?$N7C_WT1$3p#Mt}d=VDvx5^9#958R{c|i^}N{41cWQGc3KkyDf&Hluq z(`nFvh5}!0n2FCP?i8z0bPBZN#6ltfRkLXdsRE}a zKjW95|KM-v`iHb449#r_Ou1>h0yN>jy#Ry}_JHC9?E(}?>eLHf)Q^Bj$(6y1{3PF@ z#c6+T$$GlPiLQ>X5YRV}LR5gPlPgR)%g_T%G<`u4E&a&2?q$=yqmiTF|LBu?!6=e} z&d%l}aKQ0ik-tLh==z2ad1%NhH+B!O@?sj1*-`~`rEsn<2SL;2faW2gXarI{sfOvJ z%^-{12SSQXwXh0z9F0j10SZt%6-9;}M8w=*_*4-8b0$Hpf<_hxTx*xZu56D*C!`mZ z-1&IWBf=lKbr&Bp6Q|{K?2&aU&F5&j58z;L#<$4IKq1s1h=)k}`)?7aF!w5}G}u(k zZu9zTqQ|4iprQzv%feC_LMl|RQOX{*0Hy9gNkGc5h%nd&2_k8uTbevroE=vBd;Nye zDLUak4?u!lF{gs%_h;!n_J9YM#_`B6bwF)Tt13{np@FLt)^c)ku^ltj5@oI)Oixs1 zoSZ@6qZPbsJ;!87%VVa;7+Hsa5@I(XSHA~}^aVRpP{%hMk|vRn&8UFId|;e}$~IQq zYJp)6$mMC#hO!6@n|!Q*CLRH>p&hx64|e1O0#;88G|2x0sJI@kdV|+r7BC0|K`UdOG72eKz0Sf#jaQVvDx^sQ5718-8KU&r02MA zl4;i`q0<~jqB-BVy*!}mj2<8+DU~+?)oPLrzZhTx7{!w)0PCx#aYI1YhTQx7#G?5n z0fho83S}`-@V0jvIqukK^YUVvh4)0mT|f-ii-87|2VBu&3&?slP>E0fwN@xObMbZz z04ay2W&?fJsLDzoWUGU~fs0QIYF8$n1;y3xe`~=2lQ0}qAmNWaR1da&f9+um&|WEv zx{MCTDqR>?r!g?qdZk8MB(S!9Q(p)xG=fbl<2P)cVihZ)9U*nh&pfovP}Sb;FY81sRkI!OEsIn zq9NdJi1L(6&u}v|ft0!~j%YFw$qsqY-tlTKl`}*$30MB`u1Bix?E}N$Z>mozY?Yh; zPi-sYV^Hl&iAmP^jj{N;kMDgV3APz3;YeV_fNcqsR8$y(`44r?rqxPlNV!)Xd=2fj z3j@B(3|f;yuz}`{Gt+)ZBp0!Yr(38jM+>fgdvYiv+NcSCDQYotl{ zSkguYS~U~x=@rO3nmAjx6*8Wrn}bvg3U^+4<^L3z9u4?LK*3B%N7kix2sF$LVW8Cf zO*NVzy2vcu)K7(iUXX*<;h?Xf{UG3YQTFgXA6lpZ`S33_K(e=tePjUs z+kgr*Baa!p7yywEIiJ8e>(VPwEUff!hL|pToKGcHHeK z+PAoVk+=>H;u10s&!9tRGoNnosQK@rA3L9qBi>`+mp2e!qwvJBn?GH;-1bCsKWMQ3n3GsZbGNfqB z$aZYnMSA4QJ!i*UnX7Aj=`o3A##ej>MD7lNUhmR~P9T(jgu6xw4h0DcS=Fd^iRg*<)9y>m0 z&>qk=(`j7kI#I4#FsYq#FA`%8dNa1e3~ z*8a=r_ljOWB+ps>Q`QVGjuj}z?awF7nj)> z=<)~IzWi-H^4o(jw1^f*?pmljuxP#TDj-Hh_A0{3>ye^e+KcxFTKHg7Fn(I*zo7^w) zW8zXqN!ifw!VeQ+0sdqqAKUA3%SP>8uz5tsgv-@arXBpFHJ!0PXa=4OYH-uu(%c)&RjY<|6ic zRB@5MwFSB2oV!7xPF2^72P&pV>Gv_zX<}7L-=a#6K&q{+=ui?<%A5sL;6nc`TI<-9 z^Mo%it@yh?kPv0Imfh(j1L?P+fd_zXVv2S&fb~Vt4k*2M0AO;)c)_90o&BPdW%noa zU?l2qcCA}HNbB?gy@0jO-5kSMEJ0+aFWF7YNvrJKYhfqt2Qu(G+s6}V!KR9_d_t#E z5L@M}H3JVrKTp3abOx5mtQh0=J;1&(DlSF5t;)GLS@*i9x#D=8`7CMa-uJ7Loz>O9 zddps&Gy}S|Fg8)#RvcSjQRkYIea*T^c<`;0_>-gTg$}|M7cG%1-vecAYQ@G%H)2@I zQ{A-v@+Pi093eo0x^ryNjpnm_^0j69-Cd~5`oYVGS&8NpP? zYtzF64B~fr(gsH~q^0`|(cZ{sKd%ZajFEnGKh-6B7QS7@ZQpO-Fcw%yl$w<(4 z?177nuFFsNQKUJLqekoz-rLwhJImWFtGQMZ`9P??BNU#n>RI@rURgU==1M*FyZQHw zqC6duh!Sg<`rxF^MTL0BR-X`9LriAbjr9@7AnQ20guC5c77w7axvD*-=fZ;5Z7?G7 zc5b~qh^1KO-t~FDu+O*l*C>1R#2v8u8Z#XUJH?JGF<$`q@@@jp|=WKe4y1a#cY z@32rW%^e8e>&2kW)PQr-{)ad2r&mVNrNm{{%UOoqv0t675k6Nrsgp54C>MO)X!PS#vUNM-ffqh2 zJ4LqB&b^p>!L`;`xJTAUa;NV_)6H6yZZi2tp;B#=QjTK^-f#oTyob5fk!pAQkq`$f zwed0UTuQ|1h=}T^Ps%<#a)JXEl2eG{0}&j#OWVDk;_T z8TR>Z!69k4_LP=%Xs{}LC0-L`50aGT46Cm%T5ZjmQkf*EvVod63;utqa~N@ZY%iY> z7ceC!Z71u*Z!J$M5(yRr(pR@-u?5AXnLyFNQy4@(K{viWnsud^1tD*=s@V8#6(w^j zv^nZjTVf^|^YFaPh7xc8J}VtMaLr=%<5bmPwXCRG$hqtTo=_@;-?T}J6ra_Lywugz z68zeDW)%5B-zGlm?&U1EWqzMKsm`;8+R>Ch%;=wwrbMzlZb39=GM}gR1lC*}?XC4Q zZ}eN%mwK1+;$??%Xb=uIUkBk1B@9XTB@z_abm8euA#qlGXl!Q-FTI< zRI;uKb6ITjesl^=zkWq}&?$5*b;>i{Yn_sAArv0v^8Oe6#y?MHAJzAHE=(HEhGJY1 zCf^O~#R6QSA88j|lKHWW==&a6)pvMG(#syB8+5s3Aey}NJ&!M-`)xpXw`tH}(jjPteTQL>O+nR44z$i3b3k`ay9%s}&k7OmffvI8{9L7)%LtEqX8 zUS$zbRkmAsfD8Nef^*y{A_b_=7WSwB(Y*+4#jLox;OW&3{RMBAr?w`aU`iZsL}9v% zs2WDmbbT?^in_(oOoo}uDDpLB1?7b2?L6t4@) zwWvm@i^$vF`VCbiqD^P&D!dRv`0(woPJAPsvr2BA zI^-^r+CJ|vcRZWPY?Z!VO{7v{(mZK$XXVDzwV8&)aAyIZgAaLc+9|}*6Y%2xeg9(~ zKjjXlUI`ZalGo?D1C3ar93qhTXEWEM%Hqo8EFE$ssOW3aYB3Wsf3~fo7;WUyEH5gs zo83OVxj%im5v1VFsI+#myEW3mHLy>O#NohWIHzRrYG-TP-w=6bm?}x%!Q{F@H@$nS z>i!^S^`OpJx*3u*6xc=YLnJcT!fo<*z ziwp`u+7Y8eZ|Dv*(lC#&>f}RkvcP~>xH}0(VIGpGQ8icbLguO>{W-AZ4LC~ ziqE3(7ItllyUFN^)IUOJq76gF&P~om6Y8qi@f+gTP7RjkHq|Z%h@tKBqO-bEJa#c> z+se*$d1*L{zwn!Fj8BGYzPp=$rgW6S3TblX@!>EBOnq!)L+nMY#ZQR0w3kUXCW;4L zky#|%0r6+2i|4)FH40~bTSO3p>F&+>0B++BlB2VPlMwz(k`zkW<|IX^2HF1w_X z88#s)3oXGa^++IuyBCqI5lUfun~w9yHUCCQtM~^AJi>G0+vuw{dl9Ca=fZnGP5Z;b z^JdvuoQ0okP9vUrh@DZE`}Av=k6pLH2kh1Dn}*Ax4kBnnMpl5{Ldq&)7m*H39I>faleamqN|{_tA+SzqDO-?8n#W3>fiH4Boeu?*dv7EU(|w<+~rovsuH zQ^H6_*u$`fGu?s09f9X$?OB$^Se7pbR$Q-%exnnsEz$nquGYhA!e_3vm*vN5Kd8QB zQK8(8n5g!;jm)x;5iT^1M-Jn3g(VA|P^{CjYRjs5ML#>spl@c1lEOTm!aUqV@AO5pdcXIM9xCs#zZlz>)`-#P zI<2>UM}OdXSxC9fK!{mYh<>WuBge$DPXTt8eqLkoB>}p+*KfGr84(|TXC^j$0oRFq z!V~}JxR?_+)MBj9Tk{gQ+t_apb-$LKGx&Ng&Z*Dd$tq9R42y(yIi={Jb&L6g>%C0H z26N41L`5FH6f27*Y8hxkkrXgXsR<7FIj)*9kcnF%p!?4GYE&J5Wi*ZZS{fog?Avf=?~{@^qW0~_;4@~ zp?h!W3L6xfc))bMO)qD`KUQVERWDFyU}dXFF8f&Z@eZWI3UaZ`RDD3x+qOy~UFI13 zcnt4Y(!(pTpcF&hPNyuBPQb=MGrq33M#ap&RG3%W!e`n4)M_XGOQCbIR$`AyYS&)p=YJw10dyp!o?Eryf33OI z;}cb558-{*->gUPjo=XHj_X2tQKiFquQJ~bp6C^^G@u9l-*v7A+5|$)?RRVzdN0{3 z)$t)V{%>B{4_EAu1<%bt%39yEp4_I6y}?&^pd~<$!G1@{vtvOYEBzEWYRCxC?VUQ+5ICm%~w`NFzIX{?A4w1SAG|NlA>D=oAu#_)}_IhY>)##f5sJ|EcBw$ zI?>^`hc0@Ha>$-b&|7aP)3l4#wzp2r*qkdSDK#%(D9KCSB7(Eb z)vZbYhndhN;WI^NPD(!!x6D54y4Q6$hD_0I)NN-O?)d2R(@fFpJ2K9W$Wid4G1|%6 zMB;M$Ow5^+mg^zc*IYW}{dvM6f5fgdZB)Z9k3mgEHIP{>-4z55c!Rs=HBSjdX_v9` z>J^SRf4ck+ZhhlE7E=~5Zmtns5W`@}!r~3C=7mdj{$8!GRbX=IqVSoDGbcs4d%P{j zX1m!ST`&<$lbLQK5fR}GcSAMK`uor7yndeNI@AN3KUZt_C=F3n9>btfVum~_V(YNT zWHV^Cx|4e|)G)W{Ak|vXINQw8M?5_sMb}xK1ybl#Z=_Tok$UE}t9K-KtY&$CsBI7# z0_(i-bmOP)qf@BE{c_WBa+;#Q?*ZhH$Sg1d3`yh zeD}IvbTQa)AbjSexuqpJXTbM^acG|V!RLiV7gl2V{(czPC2}QsJ&M>ap2yO*rRLGy zRTH@Px|yb}&^C}DF9{#DF*nT$aHJL zTW9UM!64h@YDHN<)4KtX;g3`?Xg! z)RW*BJQDml3@3}lXV*^dvX0M(o!~zv(Y?_Yq?omxaMu8TTORtk-%4VxUUPQJMtsx+ zNW?Z562d4_+<`FaIDuxSx4GP!x~X{gj(~xF~Wp!<~*v zRlrUF)>1L5Ea~=`Nea6#Q1G`-2U@O@K0-mahM&D9DA@-r$hC(1Gm=0JaJ$33YnV%l=Y^g*gSJX2C6(N=(H6_O+ax0{+E;ef&9LzG zPGl%M^K7bg2#d$1#5Km(@Y*wi4XV+tXsdVK7;Sm}1aX)yd&Elv80V=dhtSD=3CnD3 z1_5U69mzlxT%m9@>1Gs(8GxOBYPOHLqPs0Uxi-aL{C)F9(^Sq5C$Gz=l6L~jt&8SA zkJqO^$my}&nItr?6enpH-|G%L3@fj_bT#qLcZdO$tJ_OjQdHW?_g7>>k~&Nz@s1|g zAGAqqr_efWx0lC2Nh^b@3c56~piF(Y`fakI;_KQI>cPUla5ye{1dE%&wD+5{mG`DW zi>@rkxk>8Auoj71MIyhnR17*l!}lCF^Exz~g<0vh|4K=BiwJZcZJX=8a`?6jHkX!8 zozMD5e*XL{-`S?KeSVTA4wVV*4-QW@{J67liPpkeR#c?@TJsBHZDl|uU+|@NioTve zd6(;CECYPv9sxg#Jgj+8fC?t6;2C^x64OeovSW_=7}Zys=ktYtr1keLUS*b)@VW<6v=t##XQT*)TYaOwWi9}P)3{(;A;2r0~? z(*iz~F;GLbMz~OG9LU=xBbKboMFlSdOwA|68sGM)q|P(8`PKi!6!WX-i&jZshT|jC z+1Z!rx@p&}Zk57Y?JL$f}-Z4 z)08P5%aWv2(sVHA@|cNdWU82S$m=JTNA717#j72LuyDy}f?`aS&>|=;o`N;Y5_?$G zcf14r$de*G>Ir-@m3o;I@nwY~kMXKkbV2Qv<6t zA9_y46dHcdR4fX`&T+?F#35_y|8! z`hw;|576t)I|@SKVyE8Y;%$QHrh1=R7~aw*-~!^$&5;kuLfF8%Kbt)^yVjs7(v-`x zucg^$KA}QtwDs6=(I->tg-y3Ac{~2=Pu{q_SpnwaB`cpgxXb)erMbq&@CR&*$^C~E z_khEbl&#*MaT#@wqg4+A-aL}DDzj!DDb+JD6XE@7Tfn*kKceCui?phn^+n1}J#4%% z2`V-2V;*JxuT!%Tkou;%RP%CS6ue7pm>%5-liQoE3y8d4+?1L+{6RPGV3N80OR%ja z6Fs_tk@3*OyHLG~Do<)`s@5Yhp{AcPZRWY8s|ZdXmvg0N88XLC z=kyS4wtkeAeE9u8y#N!#2EEjcmpqW|wXl_8S(||QleP3n(SJOHsZ-B_FD;Wkbfw)9 zGBY!^+&Lb(%XZ#%0n(tUftatn3%O4NzdrBDW;M68*d{+ zC4r{d+N}6szjyi^tj4MOmnj4_8>Oq(dg8}DaU*I|)^J_DsK{RoS)qR{R`B9qz;@>+ zE0=D#*UP_3Oz}FsQE_tTXmI7M5{7;v{KwT(D|pF#oet8-*N5ZXVX8Z5F(3ltA?vlaI$n;{K8#m0p)t*2cl6&^L}iHmc+eH^^KH=pyeC^!haL2GLm zgE5zw#l~gz;}6yAfs&w6FF*RF7iIp|3&)KNj@w@GzG96rCy{UvkMt6Q#czn2-KU}? zB8O#hJl}Amo=AK7+D(BEZ$b6hq~vsQ^NjoB#~%;Y>`O^FUtTJOmB$L+h8en*$+N%M zqs2q&y6sVyU*XXAV^5#wtHrPxU&@p2lJ=cR447#}|Xu+>osm&w*sMhykGtx70uKeo9n?Acz zzPFndxtE4I&F14%OCMUa((!(HPDh8d+?5K_6P13>cr2u9Eo*9j#egR!uk=;PUB=)M z(U(Dk8z_e67T0J8+e*Wn5wF2!YQ=EmVA@i5G}?V@twqrflO3zA|3%)obDe5{)1GikGgWSxSkwg?*C(;qa0aGL`L z4R%y0W~H*9bS%EFRq(&=S-y!rF*O+wwJU|)x33^Kdf{XYOc_Z=uA`ru>*U6ad?Y77 zS*qHV8MS=E*7&Ra{hd@P4d_~H2~cCy1>3X2+-BoxTblSHw+6UxJkF&xcYz*_{!LKM zz^am$+q92?c8isWXH<**iEY*U>qqP56+HnDM58kwH7FTFxGgUw8iR*2e;+c2(R=gD zN$^boE)0*QLm1KJ3`D{T%V!IT2u?!ODy3c0RD`$Rx`1`Hh@D`XQFkxvQDm)QH({v_ z-^aF6Cb;9W<^K=Zp@hQ~!jv??nC)ijKcAlSQmk#Iv#DOG# zw>xEMw0$C06J#vx;PIwiT@wGn3>dO09h+fzhoZD`xa9OKDu=Nxtdbl`WPS8-XLHte zYv~y&tUf4uiZa1NIY>E8sWr-p4^Zpi=V`)if=ioBwM4SN)?^YB04@JTU=tRAvzpc? z{mmjD-iL@>8gMm3dWi!Go8wKVOk199%;J=|+-R2p_}-Icre8SbV?BHkaHpNksFN%_ z{f@TD;cnZ%4Z=v#b^Ep6nQHrcX<_CbTlXUae_o6nF1c*6y?F%fzgw}jj!y)eCI2Mp z(Z2Cwcw}U0t&I%}`dlDU2O)d`{kJ@Y00g~m#R+KkR6EV$qj1JE6q9Bg@*l7r82x}w zT5i|oPz3|Bg|UIY=GA!-zJ47MfeG_&0{7N29RTZ!F|C{y=|CX&;McRuEl zltw8{qYLx%e0bBd@vDWQIi4eB%p)F$KUu;eZ)G_+8W&gaJ>(9&B0VQ-0#m0%QwgQ8LUJ{NK%-> zaJI!G$LU6++v??Me&K9g?PV(fAN_Ph<+qFnhQOrML#K^MzW!eL5t_=2IU5dHydawSKr-;(c^kG)S zn{C8qFZs@-3$XiOw&_AtPWY{Hh=B@ao3j*`GPL-TG;~@+u``8H-ShV@$6lwOBf;4u@Sb$P+np8!r=T7p(XL7^=Lz`0n6K~ z9sdqEnbEWD>Oo@b!>Vt%n2;TGyym5q3UTXJIedl2C&_e8W51~?i8h@ z8$?=4L`0;d1SAgKrF1JI9SVqwv~(jW($dl(B`s1S@UQQ__s00g;NEfdDxCA3y;saR z*IZA(QuI;d&9nP%PM$ZM$j%0%H;y*70Jbr4`}}x3*(#{1*}2K|6f#D!#;^THZ+?|e z$c=poskiNmY;xa1m!*s`40WnJb6YT3L?%(1=)-mZVf2>ahbJ?Rx*4g;mf%TRAF^Vk ze)$nMI>4{Q%@M44N~^`r3k8_3y8TEuRLmqz~}1pIKw0N;#$HncqBm zXWOWv&&xUz{Y{|QWL4=rCho@UEz7!#(*e6I7jGv%m6;nLRDaUnbrt9wFNOa63RMStj|4w6Mdj9Wq(IW8(dR z9lABsp(AbnMyfrtb=2MB*~P@XuIYlUc$jm354z^+lS7!$kOJMQ`+g_2{h#&Wf3MKr zwk>Q+#DCxTgR1j??4Y^PywQ8uH#1gmHJai+)Ecj>=o$F%@4|xpl?GLhMce?<4IaOZ z{!aQozt21j>+4Mio6pm-1I~oECu%~p8Heve39T9>Q9N(Uy!N7D>A%ql@kw-_D&ILY zp1I}sPpb<{R+tLk`bkc>rQCn~xD;NN9Vi$7UT$T18BnbK=-SMu$;a%7Igu_;LeiqVa zWsT2U--$5s9HiSQwj_PgYzAKQ4w(@EiDXHI5n z;$E@8mP1&axVU9iL^f(`>caHNFr#*;hV}%4(p0`WH}H-9y7*v4^Vn+h&+Dog-PQh# zh?g%(K6{g6-Zo%q6s3#1qfwI*gX5Y;Vo`^~mO#UnK=Q1tXW(H(x|~kw*zK!~>fD$* zcKy4%7F?~7B?B}=os{uMMZyuS)g%VvhA{+PdVH{5T@ zxJ#Cy&M>%g#&6PVQS8X1HV%#0Pw8mCYe?tjWKV}mfb4CN@O2bXP7>zr+y*Q>CpX6I zVo?c<=V)vpE^+$v^+E;EtYStGGmFXYecyPi$9>P#G=6kc&;NMVYqHTtxWOcE*bGLg z)U>q18XBa}Kj8+m^x-~=Ul*~`v|-)aR({zV_EHU#1kc`7OZ+z4k?{j=$_q8hyti%M zlS|jfmhT10`kwSG{Jb8jowtZ&)74_vzCcVvavTeGXQKOVi;P&|0pqJQ>*Rz5Pk?qY-7j zH=Y?+ilS96@#3`?OVOy+o8rUiHzwTf_e`Xo>{fd!XyQ6Wa4~yypZ4J~X!9$mYh#h| zud**%$YC@w2-K$#Gi~nR*1fi9i3(>{6|EUk4`vzG5|L6#Msd^X^MHJWJMjGQA;-0A zpEm)|Qq0Xj4r1z;ny+P6*l1~**mcH4@CBsB&a_@`25B8#-0H5|uv>>GXYdbOmNRM7 z2i!>t*Aj2Tzu6MX_6z$?>9x0nT;`1T`>Sk?*jnqxcTE|$hq%g?Z;kK1ws<#R(mK)b zy-bRgvAw>*5*}EkmzMyUI;J_{m;QfMEC0Q|Sp`9HX_N+S?-Uua-iL-c7}O2ns;~I` zdGZn;AOD`dJ~o=OheudO2K}g=Xgq^taKPEom$^BnoSYmQaTJ4__?Ug9De{IaxH(JNiFroF1>)%+1{?5Nc2+CoJcBP|{s@H}foOA4 zO3yAFCu?c;Hmyg^X_KfDmeT&evIV8m15ak!7k^q9_ZFH7zqz%v8#)}|FYvZdnu84v z*EsduYBrZ|7f!H}S4fm&lj#c$aSfm8dN1dy9&9xh5zm`8Qts*>t?6#wygBFkbmP{s zl#~=5Xn?TLl$4bhK@s1zx7ePLnAq<3;=>0nm^+CD&Go^!jg=LPm>5-eJfra0DLf}I zK;ctRRNSPfEp0lYgFn_c-Y=s3MlPTX1nIFbr~l=JC=`mGZ3@>Z(qngjy24Wq4Gk^p z6%-UanZLZ~p6*UAfrs=SOPsufJwZ(~8l|cV14Y8)OYeVmr@eFHPQoLL!X)>-19y4q zNS!S~o;&IDxKEeD&BUHbvO6A~W$9$xu_t2gvf{Dl)~4@yS7s)68yoBxw3|Z`A8NeJ zd=US$$hCG$OI7LS-Pn!o7*BpzG8LteNT*;HHFt4u)qeND-IBXYj`P3&nwc8M6pLtw zx6?7=ZcEj9Y*IN%vaq!N41~->15=0zCzeV@6&1oxvoHchLQUOm7~Jo&zoh(UdzyNG ze?KxNM);IBDJf|YMwz-{fOPYp9Lx`P0z$tJlH$d&hmReHEXTaqy1&zC7}c&~-X4`> zp~eflrd*Si#Rzxa+{tMk4#(5SwRc8q4r&*_UBd(F{E63_<`0{n5P7l`dii!we#-n6-KQAvYPRZ5Q_BzIuE364> zFVoVZDr^RSSHm32rZ>1GEG#dNn04FPrvFYS$22!Ja82Zyb|~I891|^9B7qDRvGXo%%I3a@8m+jfkp?%bkn_zCQnOXyD}cQGeAvgwXR;;eR>RiDMiglGS!HB0k7?Hi-*m%Z$;*U<{o~LWDwvaO z&sC10`)Lp-oid5j2$SI`Ft@4W&-a$KB>yeog~40jFKW4Ual(H0y3v6_+#SouZow=}nmR%X z6Hcz#oi~Ye|4)?1P=Zq;%2N+y=Je^*x7KcFgS_FYl0Pe8xz-M-m=+4i@~^$UirU&l z-+Ftsb`HRn=Syd2=hl9EdwV%RC>0^em6amdFTGq{+h71oD|yU(uEGD*70e#6AXbZJ z{Yd7k_1va|qmGU?H8s_E{Bva`O}1rdNJleW1owD%Ue(Nu1x~k5dBC`gzU43le4dfX z!Z%v``B4(+W(GkG<)IxR1}%IAXM7ZR!?S}#veNEKb0(rhCfr}5DlLrrcso1$biEt6 z_mpah*?hefS-B?M+WwsUU*%;zv*1k66sKbOdC`2SI5og=xXF0_8(n8~G~U)+)7kSm zdPc@Luq~+{4}byWhlQQ1OPw)|cTzuoynb?aHsPxortlo5JQiW>)@p*2i>m@-NI?1+ z%El;QkUBa$(=u63z)h+zFj@4)n1M*AC>KudN|aOe82iBjgS7tF+A3g*M* zA;H1FIMKjVChS3>{@9tc^9y#es6B}B%U~w6eE)L&UGcP$CAF)yOEi%>^O|;L+($gJ4+R> zQvTeIj5ztgPYf%y|YSqaNznCL(JoF|1b4M1k-^yZnbx&2kOMPAWZCH~L{oI#Ck2`6& zemGB+osW+YX4RP5+S=>_f&v5O*x1?kVJbOw2GwslcJ+P^UA6yzuj9@$`f)WIyI5h5 zE^uHSjw!e`@UzQoCIX6a3O3oUBfF5h*6u2O&bPelzTGo0jT6P)h&=tnF}~Fjztr%c zv1nYWmujwG@V_TU9v8$;K&JiF!(EGxT=C(0p6ZL=*gVP2@yxeSzfb1o<_sFW1^JC@ zpA{AH7MV8D0T-nVM_1D`{t_lqoFR~7-=Y$JSy)IemLfWrj8t@?e9=y*ZI}1s5)b`* zS6tcm`4ZG7j|Hi;stgI&MQA9tLRSVc*-^%5^(Rtq)$_mXe)uf7S(tcYVT6aG(tmk^ zdV-ujOynGTWR-oz5NEa%peJhInxN)4wk;ckst#$D)ZpIZf`dda1_YFvV4JqR*Y=-C-2}Fi&bJ zYu)S*zUOo8!tUqqroC4(gbnI%;64=EEgRP%Gv~YSoI%g?L-)U1aV`Bw1t)g4x4^|g zYB*qT;dabiO~9F_l8Or3wQJZgibO=jAkm64s<0+RI>N7C)ig6DhW`Xm;|;e21r

JO^QmSkl z_5O&orIM}m*A9h$9>th3f%VVA><5-g?UiK~wJZsPSt7$#HB(x}NTLX4dwW_M1viHC zt~&X^jZ;%oYlVZ~->mv!#2oAs)HF0KCxG?zi;RqXW?I?}&bZHmgE2hKl2THxIypI2 zbPE@Cnml-bg{Ch{1(nSer&CBF-eAI_$+QUo$Yhy$NI&viBUgEwwW3_wUs=w)v(t3t zMK$u*{7fxW2(`iXG5eRdBjH|viTg16h8fb3`g%n*+CjKWbeU97 zH%f~2^NYvJMM_2p^_@ux@7ADZ_rQ7WV7%$y@7}Ki17c@Piu+tCz56ibUN$)K?oQ&Q zqi+)v__!1w2BPK~(`p#V$Se*m1!=6Q-atAwx{|5k;USuvn+tV@`2kW@dFOf=`3p7SXVd<~r}h&R!sjVpsBz-Rw$RcT68NvF!|KrjjD>!YiAWT>hksljjBa^TEr3hHTfgHSgBlbg zHxJL$;iE^7BIDvjzv01rmV2fBM7&9SeEh28C0K0;`uw(f_x}CwacU|mHhz90o`)qO zQl^SNXCv)APAOL6x>q6}U5X2>h>dD4xeZ24^ z`F!0Q()fL~kE1#*GTem4KaDt+Wrcbz@9hR%(u3V#SVY9t%fNN_uY-f#Ft!@k@pyNx zsod`vH3@luaTkG+;Pu5JpUe`}5w#wh$q8!NB2US{1ZV7RH^wtCam$6%=V3|?PSRju zfekaac2xhJGfsSGw{a)bchUPLYnuKN?BB+?DSbW4+P|-BpR|;t+wQ)X-;kx$-)(t5 z_2x}iX^?g9r}UYDjAfcyxoI3g6V6-Xh(<`k2lP4x-kb*D032lHhy0GKjfB74H{JOf z+=TnG9cvx>mxV(JHPR$p*j^$ z!BDQsR6XIF>AgK?huM0{mh&Te4zfETO!yz83grBbc&!IAacyjD63+-2BuKhnFn5Ff zQ%A%5_d6-Y|DCk)Yt8oL3&u5W?Xhc`GhUF5=c0%dX2=;y*Pp&<*GE^WI(2sLgT?Z=FH4Y00Fu)cv>j{03FLXcG@Qov z@yKUYMq8Yu&bc$+^5XMnp-`o+gQGjs$5NSDS&u{OMD(v0Lx41jv9L+q6Z8XFuRHFxr49B1pI zFMRCQ+^ZSV+CmibHzbozWXsXSf7^G|#AHzGwcLxF5w%(aEq5?s_pxw*NW zFE0W<>T#oqn%W*69XUHY1GKV6oApb>*H=o+Ve%Qw*#G8^N^Kt-qXZ+1)<&NL#OS^L zky7D%Iy{(SVERQ%nP z>$N{9a^1Z#mpV%N(2b{j`;X(y^^YCD&c*l6Hh7hY9bVjP3Sc3_)e00Fu077AH!Og^ zOcrG@d|sB51Tcd)xG?Pli$1|qUOz6ZJid|T%wNIK@}@_1F3ZmUNR_)fyRR)aQvT1zPZj<(E%zYWVZvvJwr1nP$2H9= zA7ZF)TAb`Hek>{ysWahDV!M7F=gXHbTS@}}PPiQYdSGQ`MYH`F;4|vS{-m6ooQN6Z z#wsaCy^)2g<>uNAiv5WMFDTUj{fJ|2PNSP&?GS(Hz3DFXXRktW@F>gquP<%&wbf<) zdz{Ir`dHKjeivYPFH*HTVP^ivq?#%qZo$m-V)pp`;JaGhm#dAb>n{?$E_m)cWIjr zG4f0Mbx-~THG|=<+AMn!yU?qWOCcW=_xf0vCwoI{UwKd;T>3FFJJiKCE}i{!o%?z` z3NHmp_^;o-Efqr+W`M9*4%iwdCT1$-4BEo#YB*TqP}vM*eu0{7abckqHL`h_Dr!#^ z9UZ;?-xI-@>6#m-4@|htCMtU}r3%Ff99}v@e_;=(aGcsxouF*Z(vCBKA+g-PVFJjJ zw3P*`wlQB@X3~fK`G5(Xd->P)XEBcx&z4XXq2Ncm2qhCVTK;?J)lTyl^7%?1G}D5? z*@YZNc}=_k5N(74%zYVf2|i^9t|xPgO%7Fy)Hr~upRB#h8;C-FexwSOefUxa=>pah z!;h61<83{!3%}cnyu_(Yo&3PK^dZd5%;uqOWh?UsH-XKV`;%#@gmKBQibapki%KW0 zktgrd%dQ&V0ZSH`e>Fl!u{Unyv_B3WVP4m7-u#fE0>$+}a7MwM)1AK@b1_j`pOaMH zP+LAJ(^94F`|lChD(v^ya>sO9822>2=*&Qwt%XVJ*kKh>CXvqBRnZaY%34JDTK3$} z8tR-tryR5MZ&{PN*qt7c2+Het-(?mmC@Cp9S?x&QFjMVl^bhWaepG)D_+oi;wnLnR zN?D#di)DB=GfaF0ou-WgyF9<^;!R)TKWwc$&M~N1Zj&f2$2cjnnl2|?)hqOv^vj`f zx_tF~mRci6Z}U-#Tgl0hE7{_$WvPWr6)$;dn;V~F3ik%WIe0Hu$ney8dVbA5vs{S4 zEtY{aS82%CAmcPAu_!puObm9@`Rb;L?{}%7f$Y{kn72JB-GB z&0T3}X^ksTBpMvbfF(+=k=O&nSOj>@BT~p4mM_qwb9~61TlqX$tw+eB6GK zT^IhrYl3UP+Q}zCz^Tt@TC~~c;F?n%008>36JTD@HZegBpEBCBwl;SJOnU{FZF#K?)nBrz8pvtvAZ zTpwb2!^>wW1NRi}XIUbSpnWQFxTj;xVbChuhgv)hG1IPR{EmXsLNurWyt&%QkMq0f z1%;?cl|1u`kqHkIH#fJb%dD)d#l=PQL$~ek`S?8b8tUqB#fyWfM8kj+EEjp-P=LiZ z(QjfkL}GU%kq&u=&(1k-i@ZKD{XNacaOXyd0k8F@pM8}fl(6pZi3{6zo<}k*5Ed9b zOT5A9?X&V2AQP>qai)cuzQf;o1$ruHkU0V0h$(S|o_O-Ix3`y`qrqEQ8S5+vAi|An zWPmOEEnC`E4~~CMIY5PfnO#`Dmp2drSG-K0`Az`d@XN293IqWilpzh3PlVqJFJ%bE zuDjF45_o+1ZKFUkK1}<7o3MG>jj{B6mjOkYoOn{Qwp(8^q+YUK0ur(5gLpqc*bCn{ zLAE{%0#)QXHd!%-i^jRdDxIAXsukiiZImmLu;Gpm- z9eZWdHfa-Qna2FKljVNB`UW>j=LLG!mSAoMk8#!(OVUMRZL(lE&9yDJ*HHrE->KJx zP0IEMW!_n(u6m0X)=v2?)r=WM4_VAt5t0Cb8woS7cs&KXf3MTWW9D6ZpLRQ zi31;*CN8(CR9IY_ltD3zeIBK+D%_|<2+0YE0n0c4waahhvT6pwk2(%C`B%_F6_7+U)_{T=G)stV;{%^B|Yea-oLh(UDV7}=TVud zB{}hv`23_t%|C;AVw|h;`;3Q(W@a^P7-|Gd;FmpC`jYc7Ht@P;2E4rMqcEdTxq?_V zXcp~KvlQR8{%N_m2m)j@$j?K8$3EN?bQuV~XCWpRFnW|;wcZR=*_`(>A)G3JP|}$& z{)_LftI2g14I1>6^?pwmthhpqc~`~`SL>6nm?>AQaRjU6Ek@mrQ@K*E6S)@do#XUU zI8f$Q486~>z-7ytV1SZp?C#oK+5!8z$K!a#TX5dN=fK)?>dO~&*rY=-D<7;RK3q%Fi4pYp zrB{C))BNzFYf{+&Xi5GQC}0)f3s|a|7q+sw(*n0s)IP36ZL9vcVa`R4@Rsn$pdh-` z$kzpW>QM#Iz8T6HH)g-#IF$$x?7z0$c&$C@&dtoFyJL2E;r=g4Ke!gxWN08vD070d zna{*29$a)Qa_P~K6+{53Sb&6q*dGP)F)})O>Cw`MyKj)*vfc%G^iPKqY$S*A>Y!J) z?oU^yhJ1~K8YAF`0#@fb1$=ST@o`O^e60%+iZ7ox|Vk4~~2{zRa9_^AiQh?aq-l@$kcA?Tw9ELTNMFj+14uq|$_q@<*4VZcJ~;X}o|0@{WsCART4zLZ|pl-@_> z5PTttc_)hZRdiF%#_m}c&CPf_0mX@G#^X&Ri@anUXkXd#J2BJ1Q#s^s-}F_{8dZqkMjg> zn1XGUA^U#6R;4u;rNx{<5JRJJG~b7VvSVYW@2=m629?9P^SyB7Bl zDzvyeK0cCx7pFFetL01ed>T1_39z<)1}l`>M@9~Eu9Cl}du-&XDipne?ZE_TF7yzq zra%s)m1X-gQ12;;<2u287zFA%+^{%Y`-7#jxaDxN8b{Hj*ppc?hKR9PzfkMq`EY$G zGm#oIFYyD${LY|)^=+(+idBqNWvsl%HpH&tPH0fSUIlIue2Ve8lcH;ip#=^L3zKkJ zQfi8)B_|I7bNcU{oKQPZ!l?%^suSocK``hFjfk*p3&l+pFvn0(P+;=b8E%fF`O(s= zA1M*nP)y+OojYyz=tKUWm2l`0hU5;z0|9pGI?N=1d8pk_5J2%+enq2V_o zE8vzk{qH_{L<>TMm#<%k!ioDbI7kcxI}Sd6NJE2P*6y0AJ)i&pwGp8KA`m9)cf=wp zN(q*$Dh;07L*S(8sV83}&Wi(3XpD{<6Zq!kq9t zsE!EZoAChYbZ^SdZt%P6jke|o0R}Xp9S)n%aB z(VhVz<_~7u$SHPpb&ZvZ02s^?oZ`0X&yII3Pk*nRKsODVAk)oL&&_c{`1Yk0ZUrAd zwtjxny9Wqg#y?XcA|iuke@Q?j_oYgQVV^`fVBpy-o^Ec!bxtZO*nq{mY)uMQbc4vG z9o%k7>FDe{-+^u}he-zFLF9Qv*nHLB)JA8nqX)-gUWR;g{oUELr5Tw+NwG0Yqzu#S#fgNnrr9nZw5;;l577^Rf<5 z+$Go(*m{-O9}TecSms+y;3?M%^tn&eWNpR!nMro+YcVjcUM(rk@mwA)=NWT*@z^=} z{jqfNPn49D=Ag)t@H;A+3V{9uIUrypo$ep5c@NGnP-HZ?C^SGSZ4zIhw}Q9?eun^H zRYUhJK#W+t8_WE4TM> znImL0ypht2AOdg{fByWr0QR}^*4EdMc#x41lbg#{TVIcmZ*U>^j*q*+BlJ123nMS0 zSj?{MtXy1Nqzc;*!B(1(Z{O4zWV}g`3j;=kns}KIp8dfC?gads7Pq!y;i)VDZD(zw zUsqACTR^wHO4u$&_|L5&axM_xyV1k&)!tgJW?$mInf3m#;^=vk%Y7hPP} z&tm{;*QYAL;#ZX4wDGGoAs{cM4@%a64?t+$IjJ0-$b|(n_g|y9;!blN;OCw*Ha=x@a@~TJ+KrU_uXobAf_)qI5$B+BksOiD!uaMr8!E{HczI5gMbqk`$1B_OgT zJbqCiZx1l{>7QqO<=N3nY_T+g7@z|YxtgY?65xN}a*wLA_c=Rq z1m5Cx4L^uez^PwFT^$dQmw0$LIUhgb+`D(L@m%ijI{Qq$M{n*RX_V5*M9W3k(vpSM zlP5W4WiPA#xz+g|a)Zrl2mejON*u^UYU=7s+s9DwuyJu+g#LOL%0k+r8(_!$k(A!Vk7qRWOQI@bXKzY6kJ=zPXI zjVC+qj1k;=$U-8IbLIHA{)O{gOAxb0GVfyX8O$1#yRN8zU2*ly(&O#~1w=Gxj&`2Q zammRsAtfkiY|2oxQWOwuz)Km;j_jY~)aS68OE8iJK?_davV^3hIe4VPnip#C%}bE; z&3#tS7y3-s0ua>+cocdsl0Z?7$k-4g@j~Ol|6dOSlUK37to~OI(0JMMv^8L+( z4WHL|YuGw|hAL>AHn!!wFFu>B$b$=tJ@Wg;#<7os&M9Fzg%DWF z6Fh$Wm_22(o(}Ft$3QGhrB^3FiRHlVXdj9BZ#rk9X2*L6$_!JXlDBVJ`rh1{-?{*& zW;Vdv@0SkZ5EFNUmuVl-^?^6H1}j{NfeHbi*rLN!SPHeuCp^_e3-j7t*0fjcURBd( zUrB*On$eGU@%CMQzB!^A!VQMool!*xiUZhL!yqRow^a}H-2&X+sz7a6SwlijZt0q# zkkbkbQ;(5NNjMkW-`k_sht^#leW}`|6N=Y|%vW`865u;tDJC-5w-<`%m_<>xWdQ!I zbKU!vU^>CnSmjPi!k`Nr(P&nm}Mg@=$0mxYybFBz;V8Rd)rv3Tj4kFJn zpFRXyNKUJD2#2tCki5LKY`16yV8@n|A%SXMFbXhDb+k3!3Nym z4+}ZKD@%%npFTPN{&|N%${pt_F;fThky9O7Rji3U0?rHa5cyfy*f5&>j>T@I))J~a zK4Nez9Txd2S8PAKb1Qu1C)A=)Tbk(GD#FTHt6{eZFr}}cmEhswaXJ6%0bv-aoEjyb zps6s9LW%9&dwzCYdcc6xKM`cJnpyTGFU4cC#KLkoXZ9q*M;|C=WRT8{h9Zd9?NL>K zx40X|i?Sst^c)JLftQ77k9W004A?q`bOx!o8tW#)1T~#C8eEz%e~}xW!U_SJmFKKD zxoJ^5&z_Gc)*LR5(#S-ZQ>#fjWt$F?^}D zxV|0%;n6&@nVVY4;s_U)IrP$~@czOB9L> ztA6FYuv9ZS{wNkBhrDb9p%UT46w=qw4O7k~$Qr*L-Yj2sU~N7D1q~T1)ArUJ8{eF! zE$2as(T_MIjaf1{EfG(8lZUUUlJuLnXH=kYF)`e?e^7>C6Sl!l7y(!$Y%26Vc`gBm zkWSH&hd5+0`hJcM4urUvxwXwr`|@=n0)k507I>-14FarOScEQa47Qh=dWTBR%+9hx zJE&Vw#8%Jz`p=&~QiF;AVOp+@9mN}t*RyFC`(IPi)34$s8Ab%$={)f^!b_H8SFyPi z<5TxG60X!O(6fNMuF{1G6wa+%x1b^YEVz_xw!WwW;!nqCIf@Y$aL_;#A+M(PRgrlx z^Kdws&S0mGruuPllXgvB9y@FYZs)fbNtd#$N8fu@I_6QzSKj#k-8-|Px3?SJ1jAmu zAjT$UiiH|w34~-~4R6d<^M`F?e|cy4YFi0SU5P>2*SSm_p1vkU;5gfv^2A6k-nwa; zMcHyEBfaZY5($RU$6{QT$__WZtE!w5+NNiY%l}9Va)bQUIzjBloa zCwu07S<)%}269jn78%u)%+bczdC(>b?e3XY8&?dM-5%Yz6NIwl-^adabRl zT5SKQLh=pTTZn8lu#p-3^q<2g)YjRVul=QXWMgl~H}LaEzEWbb%~zwN7DiS-P3P+Q zW`X~ba9bs4$V^DMXVd4btsVa$(QSLO{0cCxTc4HKz5uT1s27tFlVjh&nLAykU(E_C z4dMX4I^$Yrbj5Ij9IBVdM{~qRjy9P8!`b0sV$s7FQ|*m=i$}aO+@Cro8cG&#kvB{_ z&`!Nyg4N5Hk(lK)2%u&{5nP6$WwCMe=T8S1dzX7|8_s}U7NqcPpeBn29Z*HocM$dM z9Zo#9gzClF%}vqUTLM7@A!dD5vjVyrK#j!m8lc4ui^s27ixQeV55=D9HL6<`Ba5Dg zjm%lQB`OuW4id6!SOgE1x!Cm4uRg0y7@+w+Rc%|=$}_mvC{ z>2_-#r6kSM(O4G*P3~&L6R6Z)?C!ja9;-c zy{uOpH6sr#TRBv>P!xRLgt-yVwQBhIeFcVh}D?}V7 z%bxSjBNw5wEu-99gM%!x)1vyPf%`2liK!@;?W2G#*%V9RoCsp}#$r-KN!OohpdYU2 zMXK#MQk-CKfex4z0t&Yiu35R$yxi7&OUpptfI`kokK5LCiGM|8e4CnkFpC!*c<+m; z7mk672SLS!D~iW{Y{SSBQWnC-2Xl?*U_52H@-4}r&NaWs{@b=5?+rnHdZ?KY>UOGc zQ9YjwSZxHT1kCR3$c8hZ5Qdv(^8l1V$tCjz5fSnll%av|h+xAGL<9r~i-~~}^!c6g z3xSi96SIx)`ADm|dwc{kAy!Z*0`mE?cLkX8VD0XkWb>tuZy`G^;#>!zuY{lS^3|)h z5+kre-mGQ@;7FCpC)CiXZMC4N$SSBC)a+Fsl@>#EbafFl3MWhfxYXe=E;VnrSA~MC z8phQ2z*R`6k=$o;j2KfzSuPLr{@LA%7t^jzvYpT^??3!4A}Uz?AHT}`X@ZX|stlx4 z$SDWBZL-FR8Bm9CIAvnrI;0WdCrj$d78F{Di~xGQBH_Re=vDz$Q_|C;0Es_tEex_d z_-itD>l;VXxO{zG56VvXP9#xELmTcScwr8_Wa1Up_7ro#$9KYUAf==X19?E|>`egy z5=4{_*}AX9&DGWS;~>-?9Z=K9z#luvrgl@92BKu%|9}uAu<=MEBOm8?3bhqd-p)1i z=odW$j5C?{k%h>=$D)- z{Vt%PS^~seylxwG_5gR2f|?N&+$zxfH=qFSpaH?n73TXNMxY%tU0@WJlbgGq`O_Sc zdmZi06QfPmy5!x9jHm+Pg^YHl1Uec*u!9i$E-}&Vwu}OTgPL{gu1Cg;v(l~TL8cY$ zRgLQZ0)^-W*{?hbQF9}o^9a)yTGrBNma{t24M~ACV`BV86AJ4 z`sM;DOlv9#jSA4N@={e;81qso@>13KQoFrz=MNt2d@ zb!@w(DyWY`VW?;s6nrYv((dce!tkjrU>nH^0QXvf#P>H-Ut+l_n87qJaw|M{=i&eX zNIdg^G8JjTfxhWC<=U|k^3b&pk-XW3-#L1WW+y`cVY&=vA@X|A+EaaUSBNtc6BAtk zt@Q+fQ;yiw5peM~N|yL}0@7;?h*_V%RcV)AS#|{_b2AD4bByO_n#j(guX?zzAb8Dxkr?w;S`>!xM!5Uj= zU#~R$1F5gU1NXJHHH20wviSxV2r8MWw-dI|#7x$?aiN*PL20~m%kA7!jiZHI;Yag? zM3Y^>AfOeUMyuDZ zlw$|NdM8k@14>+c5kOl$JqgOi*7o)*P(GFQ%&WpB`!`cOc1wRUPwF5f}p&}w;2{g;qlb6v$NA1J6{o9vxGwJQO29yZdRhP$ zYhSH47C9z*J}sDPI8O&gUZC0)*z^p5`NTZB$?&Xb^DI=?$}7-;$3S-kZX40Cs-%luLbTHLw0LnS|vPFT0Qd)Uz_D@6Xd{jM_o~ki3~P4V6fIH>%lb4 zD5k5qM}&p3N=VQE%!kk#gm`LZ8H`HcwjxHpR`0PX04)-TTZi7szX7dsX?RUyA~`Ic zQl$pc8x07PpHVvi^DF_m26AS_ruc)AySHc=C(;p(*48DknB|||mc{_FNe+bPGyOtk zpg530W56(n+7(GWaGbKVBLcHB_cqhmGBtU=yj7ifROszSHZ#+bj1umcon=IjCRqVY z^C#&X2Ai$VuRzM@REeW|qU#Jqvz4{=61<#RV6s6KHCvz$QUyeWJhcK@mH=k8kp?5< z-8$r_b};4{wB}zUV`e6XrLOMtG4HSJ8CBy}v15*?>xcTh$nqswI=R77;rpl}lX>|ifKXWZmV3DLa^Y|+DkYS_c(3m;wmq=lO4 z$M^E`LS`nHqNt!%06{zflo#(}R7v<^kvgLBAvAo>7yo=>t}bXc3Wtc2f6RaX-U15S zt>3jS%Pc9qaYtuR>X9-q^fmK(Ecbo=Di1?+XU*^yQ)ir@OKkieV6}bCuwr2x;+TaW z9o4_%o0-jqt=50VI2*Ar(1T^|zYUxAHgs#qsf;3;V}ya_x|=Pc?yNyloOG=MWaCnh zFvC)uyL3T)9|98S1(*fF!NnEiLB7F{8dAkGuPZ!%rO7#QcUF~QLN%UAHkzDE+YF>2 z5dRU4JkT}xPz^w=Z=auMhX1GJ?q2!en-?jjxgP|s1z790@Pv6DLO0AjK3MaEb;WnB ztS|1Xa6troA$ycMHn0tAj5Y6^Oe4~8#Vpn}g^PHlsm z0!_F%xegO#R7f_5@c~?stNhh889f)7OtVJQFH@KhY>}80|#$`W$ajM|5$)p4UUOZ)ib( z@Z1;UaAh;?Wu8WVd%d;$RaGWm24`>z|N>M$of*i21v01SQcE;;(saxJ|f{zJ{{00K9_I3*1d4e z4h(g-d|ic+(WF2aRmX#*{N6qAad+|Ds52IAEy*7yJS$dZ8*;2422z$H|*dGYWN|s<1NCRbpmL z5%nQhkL|e{Tev*SyND9>w+X6>xG2UyQt}^4NoVbn@YLg6W`7m6ar=MfBfQwb(_G(- z(h0hX_N=0b!$7l4J1uk;)k02<``EA5 zIU5j7>N1>7DCe7c&bPO=5KugpNtO=gn->Y0^h-_SI5m>XTmolk7z_09vb4LqYG6J? zEnipcDK;(&nSlyZ<4}OAX$}Ay&sq3!c9gRaNjjl2^v)d{85yx#a|f($u61U9zGeIz z2EheEgj#KS0&y9<1FisVk$6Vml{@hOyAN&`#`9;gT%a)^op~SH@laZqHDOATBT4c4 zD|r@*d$;aX+J43>+*>KpJgG{u)O#Ve{_{qdX=#LRp{2dIaOvuvQJISEO1Wh>9TWlx z*$Nuha`z2gUlK%`(+MiD$xB5=#d2u7Vr=TOgD!kvMy{;K90nhe5g1^vv$^U)4`n4p zi;015zik~Vai1x-fNlu#SFjc@D|BoC7Js!8@}&cPFfU?r0P+Bvuw_W>`+#|-Z>tiR z^cI_Zi?Oe&R&tjHVhK>2qTX8V|CW@OBeSjnFn58NAn@0KP6N(B@!<)Oi_n)|zkc29 ztOuE*g5QOqHWDSmDbBHzMI9!8V@{Iw!=X>2GGAtmJE}z$c<$f7@6(%|o$U?cqv7{j z^ro-6en7rZf>!JqT|2GUtJg7LT_Wp`+yFLl@f1cJv{qz2s?)@)lM`=bh7so0(VjtA zw}9rwL9s0or7|j%#!ZVa@z7m7Y;Uf?>m~?+ddU}|(1v%)4jm&D?@#z@p^<9`WDSwW zEq`(M6#I`cJp23Cuqjj@#xIcZ3&Ui|szI@!a~sJT|nUz^r4j#K=zT3kNvgO+2q z(tBiLM7c&GN8Z?&LDmf3Sl5L-%7+>#W>|GwdBMd;l?O&BFzknh20jEMAJHwW50%(R zZ=z!-=DadqjT}ci*!o~SulzQJy@M_19+wtLe*zqc-!tz)0^_9X4x`7};?D6=QRS0{ zmN&wCIOo zO5Ti0FTSh$NVxKQe}5c^YF`5RK^wIL+3Y9?*Z~0Q<%V3K)ttPnt8g_I{34ix?2l1vv~?I%keNj5;M@V$3QSR{2J>fw;uSmV5Dhc zmK&hul&vb0^cT)wahH;Xo(O6+F_Yvxg(r(?W>TI)RSZ(b34BC7qRz8%v+2cATkA;0lLC^~b9xFX*V|bvqf*01N<~|{Yy#qWU zf{8%iOhG|mFq7@#;^EN=xxCDNg5TjsWfY9q$den?H@I)K8O;IHOT}xTQ)&q`DXXMp znh8{FHB;e{EOwizMxTbPFu}GU43Wg04azqIUw+AHWj0;Fpt-&qlVG0x@(!of>9ur5 z$eJ?1{`c_15oJQ?-iDbW+|kK+J>C+JDF)c6vC#Q`4Fo2@(E)fugVtPrG2LfN@-+&=)gsSzH z1~cip?|YbqIk;^Q7gHK;z~uY$D_7-4>!7Om0-|@QIAJ&T?K^jHW#9hKZG;*LkV1AS zCeV-~l7LZaw8&ZP9b`v9BsnU71pVSa+8Bx0;kZ{ybo#yj`!({pVC>86%h4NSc9lAX z4{D8aM1GW3Ds)oM%QmqlMymOf_lHA+3Wg)Gd`@n^MsGlIkF?fsd|Lkf#&0NvW*S(U zq5sF$dw^rvxAEh*J(EpV2+1ycBr_C3_9kTBWD~N5ijY~6m61Jedy9;aWM`#FWGC}~ zUcK-8`;X&)9M5|k&+|S{_kCa2^}W8|^Zcxn^#>A#^y`;g_; z1syf4iB16JC-$I|DKyBp=#(!_Num4gZ0&cemg?8ijU>+N+$h^<>68z(EFv>6$ z2^Ey!2WdK@*i|{s)cHOvyMt&JkkXf$yb;ENH!ll5tds8d8)_-}6H!ZR{Gr5Ea|wzB z_K|Za&{f%1Rm0enCye4`hyw;piTX|inV=cPBcYv#i44CQz~S}=v>r7vd>&5|OWusG z0`M3LVJIYOYHIKhHn+Cgv*g1N$)3OQ#mYd8BeGGax5SqP;2OU=e5iQ_(96RU7-*KQk&0QU=(0~VHgXu zn)-7KT$)QU8&eAJQe}8S-vXhL8cj;%gW8@@HNy3(2kPQNqfnwx0!$_3gB#bS;od{J z9UJ4}vR^$UXslHGK;QqsBde6i<92J5d6~-1o9`80k@!&R6pyh(B`T+32(Eai=3-h3 zSRw>ApuYez;UWOJUZ^&p1rR+KLN6d9Vkv;D;9yaFJNgD>iHKeZ&hc_M3eWEUZd%GI z*grj+oK54OG2zGa)b%Jw4fDM42EHSPQ5c=FtC zRXWrfo@;oR4>ldiH?BRncKfD!-8p6kmBVgSk-E$L(|;cISeHO!CeXm!JpPgUMFpDs z_fV-Cxe+P*7+KdCBT++E(CWK2mPIBVgQRf_Ag|_$xg8Kb<^aka^a8CdoW5heVYlf- z6cMuvVJY2!w8YF@hI9tRW($&z3d+c2EJ6Y%zlL(Ib%#!f{ytt#Z5iKk=yU= zDFOrKSpr~bLYVM0<{mW6k@8cZZ_9ynGUcG5h0Z8oJg~Cb#-+5{Y7nK>jT;xa5{aP4 zZ{$)9RQ>}R$ziA=tus3zu;g7d8iin(6>zgU+Z5F9uLg!C(e^Z%3f^=UV^kI|Oj>Ww z<_3l*qmULCtCFvCZO<{=WEL%!vEco1vvXfPb%2)CnUNqvW4O!T%Nb|yWV><^P7CO@ zsl7L5qM-$^dd)^XYpVvov%+glU*=#wEK>Dbcl(0dJgC9nL$V%I^`{rMlkbdEt@k^ChT8Zc zw>h`@1v>ZO07+lJxazc#96yPMS;|<3=bCLMo5!i@T)xfb#IBT|WVU-RrLW)z!{AzM z03xT{A7B@(!)GJ2o0a|m!%n*Q3vr=>r~uXGf|-p3UeIaU7=v~>Cn=9eyZ;Kee=v3z&-eRgBvv0ad=`v zZChV=r123Ei7I)b846v>-Wj>_BswiW{M{Ud@cS^#*s7fqxnU4~Q2`S{nzr6EriwV| z$UXnn57`Q_fE*3Qs|*_$7UnC5*on-d#g(*4KLpI)k=eq9p|!27oSb-eRTqGLMoQSN zvPUPkr+iZ?^%NV5)`0Fq>|j(U9VT6;=}^huSAQyxmd;>aQHStDsF}5OvJ`%k~o|QCUgJ-nyO} zpOiir2)Tb59)Rs4lwkl&^F#Ov~w^)8Kh()ZzX*N#ks3S{)p`EnPo0 zL25?8q>3KNR*t^s56M8?^EdCBj(W~fXOQOJ3U+= zZ}ChNN@lo*R;k^Z{QC=s{Kjfi-le6b+cLvH`awlhQ%5b9aSrr{H%tZSfX@hl$47s7 z1l_WzviO@7f%EV5d6BlYX@AP;02E9J0sU*X(O~D#v)EWg(C>~^JKW{q0Q0+WlpawQkCUn z!(~M72(b-VMRfmqWR{A_(4|%9qGAa;utL5xXnLddEJ)(jN9({LNS`PNtsQ(n9YVT+ zdQ;YO^%LHC*@G-vS2$fS1o=6=(!7kQi=q9+Mj@IQNK&{YCRRJ~L&)(nchlj|;-Ijh z@zKAtkfLGb2b&NdNAA>TJbx1Ys>yeQ?rYlAc<8t%cY>sFjjp~i(cwUsGUpFIDgOTB zg>7V`h#4>RM*$isJx&jf4<2ZEMny$2wzvZNw=z+YBX;%s8nGgwvd@JjH;G~Bia>q- zJ(8Ct@LQi;b|dS>dn*IZ`=Eda81P&}AVBH(BQQDGA$ev^&ehSb#<=EViPWgsm=cCT zq%>=Eeyq#eeTGIPHqokzxO|dWCx7Kq{m;Nz`u3N)w8#>k2X$xTT$O18%8@Jz2RUTl zYQCEHZba1n1T4OC!kmwQl@25G{R%U|k(U{a`ipSO1omCWOHAm1%oGj)6WlnBy**$} zkys%xE~g?m-tlsao^CO9;L*yS|5cA}D<;Xrr~^U@AINDt!EJVm_}bKTt@`Xmdrc+m z)GJ?`r9l>hg&J7U;BF|6nJ3iUD#)+;*+OH6^%SD@RH<6P9CRUB(6p{C;UV+6a>oVM z81`TLp-eAf)g^G~6&jh^aUM)9dk1W|)0uLLlJ%R`G49 zQ%s;Nq8{j5i+cpAP2HN;^W6ah32XHOS)gUrWYItyOign6v-dY!PrOR6^;O5V#n6mW z`~3g30BzRzENf;2R{B*|$&J_Ww2PmVzJSAH>I?z``QC+RANfmyzxC7TX#Y9~h$uvN z3TU%#a{e_K$ub$J{7t8)!(P1lh4F2455l7S5_HTG(nVE6z zC3tnF0_$m7In>43({@!3RkMyZ1~&yRzd#_6q`p5*n;j&E4W5N!_kOuaY(lr+VOK?C zbY~7mQ6o|ilqKv72v!W3L+nW-tc7cQ7I6-L!F_IJe{{N!X|1q~cbp>_{&Vip4S%A{ z6HHs~-&28_c=~%|_{)2tjg|0o6P#=jHuYd02Y$=M`hKt!o8iS2@|X=uAwW$KZT_-1 zB{&5X;Ny>ZvgFu#sJrz%Jv~Rn)yNHXZWk`--FF?k@VkgwPJ@5$XQS(Dos2GD6^5|N7bx- z>U`S@i)c+yh`HOEvXAZcaYnC_ndXN+_4`Ih<{quS4nKEd+J=d1GYv9bvHq13oCbcE z)uIU`i;4<%ncMXTSNGa&u$LF|hPiM-Zd3i;^C-2~5N`$VWYC7X ze}m_0XyZo1ZDHX@KzbnLrsCyOpo0+o1%rrvFbHQE?A~HoLPKT$xj_3rcR9UHJWUr}Mj%_?J#vQ4P{?ol z8>LS*GMJ1XI-n(Ws+-((&E_v#;$vHf_{nFYE4-gD#}hW%f5P?A-csiQcl=yz<5O~- zXyrd}@NGgiXS>Zcg#1L}z{*6}XgFHNJ}PIqP)~x6t4D>QZwIxs`BZ8Z& zU@bv3xzNURL%kUmWB^itwHq5es{L3@1IqKVUujY(I!O&%Mq&mr0L(V z@?-H*v^*^YA<1T(r&SJ%IbGcK89Y2}RM6jLIw-NCd_v=|QGRXlyG6uyDg(j|W2yHb z7^8-sIdUEOQM3Z4kHH{f3Fmp;+3C>;ph8)5&^P{B0QRj1%1+R6JwO0u*6g%dt55|mkXE1 z^a7|9LaU&YX2+P6=tBl#qte{pS)Vz)S>PtCb)@%!y4k^}B;M$dv!33+Z~f`d!};VU zd!rjnaep+!Cpnx#O{kSM++wo}h|mqj6bd}AR=&Jtn5epc=XoJfUs32?-XZ*}*xx#q zDs-(am=y@w!pSL3-2|rssH>7MhGJ7miN?MU!*HU$wuLtG96T_fMq^AeWQyTQ3KFu> za5k)cE({AJIFHS}nfMSY%q^HI^9Fux`J^+xb`aRi7{wxDA3q)r(BI2>K^uz~H^UME zm?#P_--l_0Bb;1;u<+R0B+$o?L)~PXAoWha|Lhb3DY>TX~w$C4opBpMzI z{tO1(4C2QR|9hyA<}W$IgtxJx^|wPR>cSq#WykmN*3l@pcQes(ojg3h&(usRFSN9< z;pOI(<94Hc-ASi3GrW*EDZ>NKI#Cf=c|we2KZk* z-V1W(@0ACipg^UBt&~Q8HFeO=kdM}|L|=l5th3*-M&CF5)`?uO@s>|1?gcx4mhdbm5$J z_?px>JOg&P8^H88g4Wb*zLmJPnUvOQFn6oL;wc;6xq{9wyszwf5e@_@ndk`oZr-nSHbGTvw7% zPbbNd1yIN2 ztEY8@-c~h=MrX0|Pat#xQn5p833^HP?d@$83iJeFJxf?9&);8}q52WuH34JgC=YWD zScv6|J;#U*Utx{MqSI5TkY^TNJE1@4v2*<(U^VFYr8homxJsER((6eSQRSDp(MI|r z`OzZBxiUUe-pdfQ2L1PiFNg{VMyIF%KYPL_P*qj6g=lqw9sql7cV6fMG?x#Ad**>dmLT6|Qh{oiQlrp&?&( zXA>8vLySpPr&Q;Tj<-hpA-WU)rRHU4v5fCA%BspF=smA*x?)^Ib5ECk>{!*LUafrS z;^|091m%mmWDD1_-t}AcQ*$pkhy+qdAYT`n!f`&15`ErZ-qK8D1kPg9q9xbCLXdEXL4+sQlH}txsq;ct)bU9eMX>D}YDT=0uR#moipRYH*lK+gn z;R-x)^QS|}OQ{vxFrTR zFu;#5c^)4X)rsV*vH?T_PCZ-{SPF`IKexjfm%;|!upo%aq1Yv-q0zJJkWMZC9(D)$ zKAnIq64dJbN(wrws8Kfh!@GI2WuGz-lYQ*M?icd@ogJb7S(8*yyJTygq z!mum4mpuM=mA!jJmU}hpaq0tezSLpT9M<+dO5xH<06@5KRf!y$9nKvsTXM-BZ=E+C z@a{wlFA2_4jcnwHu6&DG9K+2oo6yJ0cU(7QJ`5{zXg*{asM$6KE=TdG6%K&zzVan7 z@Dalz;5r7n==KP-CVoW!zV~t2f!^F0y<}xJ>s-$K(K(f^%#xm~LU{s_-UVTgg%}6y zcFsk+p8PiZ@Mk`DnV`b6=L$+L(b&qj+Q&>O%82y)w-8*6qo(;RF6J;Or}PD=`lrsI z)@mwUB07H_DbC8A%ysxhvDmwx4n?S=P8 z0`t2}lbJPb4BBk{th%~-O)43VOs7yr6wH3?i;C9rH28eJ%mhOQf)+$hb}KK3T-6z1 zgxov}5oaf_yCU7QPwQ`wyZ+Q1njD@Qx0rN{7oT5>(&))*3-2cY?ml=a$*bBpU^Vrql3ga!c#}!}pv42TQUqo8k$CgG1IAuT}>#she{a zNY?W-1>VOP*}0MiW-2Ay9lH0K87F9rC z;Q+daPp*L=)de&7f}6xGT?Z}>jsv}KMOTsp6F3~Ma2Q@qjA4hJ36O(sxE!P^Tml?y zP4tEbXl9(}TFgvXge*~l9BkhjBE;EjxmrjDKc0QAk=@wv_xi4Tt|33Uo%b)xz~8^* zr&TAoP3BZTyQa2O;~EO+ocIZo_3O%9J>3J#MFJl^J859_DQQ|xHgTJF!B_$EfC>gf z1v8)@?NHecmzZF(G81htRM`!7@Rl`U$MxitwElSg@IZ@PM?3o=NVFrD0rf-lOLd4k zy)Ik(ejTUUHJM!=G=?#o`(4jn|8w4j?kNgj~0tI_CjzP-8+VyEUn%v2*GGO1!T>a!;@(buk zbN^LNBviK?DIy_;z^~%j2ge#j=38!D0$Y6{;mFBaSt~;_#L9Urw>p~%C8~{|S|rVh_vZKdA}3v{CY@6!^}B0-hZEsp zwt`;dJ1v8(biuoE>+8ecFX~!n0No4#7|njNlGBxxt%!yD_r+bB4tUuGm+==JyL0sC zTWI3=`Xd90uDF`}F5tC__<(2XT*D(UfM{mQp37pTvz?e)%Z2?`1ApE-nCW#)-1K@RhuYNjE1GlT))n*6e{_RZU<7$xm}6(rJ#aUp zp&5L^t;)wX7UG_{T6*{GGb>4H=|s~ThEFR_d;YD|p94S+mp2U zlJ%8bK|$UVV?TneU#!O%Jq_YH264107mXGx1!JFehF^DoG8poaJPANGIzg3nQVU@3 z62NXZ0XFPg_H`UDbunxKp1DHFM>~Hx@4+(%2L}Kv5*H*d0IdT!R)1DznvCByz z0=yXqY|;avDyokuoRh&J(Xrsxw&14aNmbICo1WY_iw>Xk7-=beLc4`F7<=5;YrlEqo;twW4X-^p(L1!~}MWh2_0a_e{xivW4;^ z8mG?g{(a;=Akc{`C4l(9FFpaiG=hnO>6}GQKD_J3$?6@jtMF1$m?XH+Ocq=%^PP0x zn2|)(5x{NVejd6961giFT~m+&MlQoakLOYi&WJ320_m1dQavEMv4k?f7x>8oc9OelD&Li^0!zCIyu>1?$lAcsLBYa;Cd^WYf>UyP=+P&D0aP9dP#t62CMb z1VJC4p78@k3WcsyTILeuA5?%13L;)^oBRP@1i9Vd=luL-`TO_95CRsV+#hQb6_)^e z4V2hNhem=|c9ZYRqranDdGxF823y+$- z`tB~XrxQ!w-hamj*PPIG^E2nmDZigHKC-oKt~8!T_(iH(-F5gRFM} zmU|CCV6`qSRJ6YE|3PTcOLxh9_$q4<3P*aiX9xVmbBB!#$2;`)5tEA;BjD<>;1wR! zU59M~$u2;=IWDd6IIMy4n{o%lYZK}|IcjC(1{k^#II1XVf5uH3w98q^j~`XS7}Mg) z2lw11KTyAMw|V1_$3f7NnoT`*DnGVAN6*shXEm0BBAf==_sjMaGM<09efFBh=VWiu zyAe^(Y1kjV4}mW8@d|o43k!+HhfETeS3OJZ01uU}9l>^`>u{o+YS85TpmElB>8UvB z8uBa<>t3W6oID_p=7hOuVLo5|-VQuNV2o4<9m(7*F{=>3V?Xw z->@T6d6(JN(>=i%;N(H+LIA>=VBpXhWp|)c&H;>!&o^8bdtp6Rs=wfow&&fhRgGpm z`C;`J8eCLlFZnEy9cEbZ~>;#K|fL~g4*Ho!idihjXgXodB#G7?#rrq58LH~Z@tgift7!a z#2Hk;G}Cv66$-p%2u&JvxOy4lbr`^qSJ8toy|hvaJXic5EU(?YU3C9UNibY*0p{Yk zeOF*=c<{f#9j~r%>OW+HTa855RXdIm!<+&X3SyQnA|!(qIXB2~ z&yHHp`mt3A+4OL0%HHft57-Hp*4YNMZqU=ZNRz2;D^wd@EpU;+uYMM0xv!AGQR1?x zfs>z}bQI(A5gkPfpV$WP2=Y*nfhXxlJ7>*mF2Id}5`yCYPAWnI7wK#DE28(EUZ9pxUeBw?y_fE4Xj~DIR%;i}x$?(U!D1yQ2 zdxz}6Si!e;J4_%>UWCr0C)DMTfryB`1EpzJ@6_lyoq8#a*j zAVej6_K;9TxD16#AM>&{Op&suzt3BHrJkqO;g-3d1Jqex19mi!b4V=XDIHujYeT=H zbFrPq{{YJ1S@ii2y3{YOz%Qkng=gRV{XWFkSk@Rr*%o^u4Yip#;kQqg;9;+G&&x`r zVqtj(<&gf0WDEGPgR-X=+$@1W;IjwxL%FN=Flf_;){g(i)uCyUDg6IaEG=6=M$WFIN>BJF|O%=>pJEN z;85#;hPxcZDag}ylX`{FvcOa!w)AY!yEIXR>k^EV=yN+NL&Dt~?ij9HpO16YTlE49 z^onUgPFTthL(@#Fw859E00D#%7au-2uJ2!CN&!JSL}}yCh=JUelto6O;IA|+#9_MW zNrRaKZ!;-c*Xv|PShys^Z}nt0>@S|rK=oD<#AF}r)zH6?sXlBI|7ko474$JAMM26I zUl4@&y)&(+1Oq1o+%?R#z-ePxX%kI5Tm#5-I_ftZ#z-viUBbej*(T(;6V74?BowJd zv-p#ck1+%a#XV}3cyLivcf=jRjs<9WT&-zR7XO;Zz`#Hk?91oY*xA`hvjd?z1>Kz@ zh;>Pr*}`?%D_;G6zPXft?;vRO>bLK=H~m9OF#Nx9#j$CBA`fcJM=vY}goxrF#H8e<|g4%S7|mEyTnDxC5}_qK?>8pJrpWoO=Yp=qp0@fDPp7KljB;wyYAZ5HK$1pA2Al)r+R;h z(0k2%AZQSE42SY=p`wQ+Y<#U7VCqOD90Ku0LjnY(g`M;vx+#EsoN)SSVz46<xgeRa6iS2Q1#k{Q!P2Rzv;1n?Z9Kcn>LS;P$M& zrmh4ui!r|yq#lSAQzz`)EUkIq!5?K2FA~_|V$oi*UuX|6NJpcL8Sj|UFQ@^K_(3>x zIuY3rT-3H5IVG-#-aFrbz^71)WINz9Jmu6&fgx+vF4OuZ-cTvPl$Cf8XS$tI#C z53K-L!?Zy+KD*noCL6&ngPv9L-Gj;iH18u{AmwKP3OSKq;t+EQE0$j6850%?=rv>) z1>*CSg{~Js{xp~BeDW;1Qh3iV#8Kf@NLY82)HN#dd$veY#^_ky3@TxD@E3MJbcr(0 z#6Y{&Y%YkjXmCF*yJKjuG(Lne8$dye^blV;6#8y?{#hBpfr>={lc=hx*^+<2HI@R) zaH;;s>(0jCEyX3%;6p{;J&03;J;D?yQJZH57i*4dr_o`ft9*YvUV`xxj1KSkGY0Jj z(nU$&@s?+fu68}SAUq6IxW*T6`Mcuwf#b4Ot}Qj`x0@2U zGb1fr4dQX3liwmZ@Mx#Ii4xui>90ip1}sFzf46{E#06dy@GGfT$ii$KDg^n4Yy}1A z2A)Hl3ZtT`QOpM1sw{=n_!vv@V!-Bn&~xL|%CPJs_Zs3U2c^8Ol|mE8>5T1zXW-&# zhYIc78tB_3A-3+y{BK~BsX#X?fiTf7v+yqwX54dSI3yyDT%k|Rx%fB19rO3&#c`I? zAtWebcbD_d8Nssr_bjhoiMJ(U>85`!Gch0*ma9kBUeZ#`KJD}H!S7ih#A0T>FLQA@ zvNwXyuE3y-#TB1TUdUiMaWq4J)gsq_`#qzpCEz>qgEg)&J|1Zkd{^^Qj@cg_E;b4b z%Z@98y9FV1UFvh|%Wf|*^*p(Nqn1#;3=rJkv=>L$z_b3j%)-JXnS%(_0i?(#oOI9P z$8aH^`mw;ddpFMTrpZi$2OZmolZ<`u+X$1|GoRyTQBANBx=yQnHEYhh+)#pg<_&Mt zUiP$jui1m&ulV}qF)%U@QB)uNsv1Iy8$cUK!Mp!n@%(kAc8y-qf&^JOioaHSE~%S? zfc6{>b)h-X-UJBH2#U`N(5@j4y&A_c1&}9dz13uPX@!_;y1&=`J$`=b0@2O$ZrF^B z0f5{1Q6SDMC;;qWZ*Q+9)}`=x;*~XYhh8e+DcS&yi05Xr6g`j%Z|C zBa#8W1N>s9FJx8VHOjkG8}|zNmeGpSKf;^-8svO(zxB6}Z&^s6*guF~&koI9y{|l} z!tReG86(*;5Z>Jhrq-)Az2v*U=Pp9AHs8&b+YpVeVN0-;%!+=(p$79H5Y~|Gg_`rl zp@H_Z63I}n#vz3eqR{T}jSpM&Gh(#Q$zn4n^uwH1yS>Xzr(Y%&Ff4baEPfq=7SkFw zQAA^eF!KQ1&bnqokS`L%0-b*dE8)tQ5=Mv`4`ow~oCkRJy9R4+Q7rm|6vLqg2u09Q z-7NZXc5h_FzaMbR`J!}6nCp1{z1a<6Ez8hJo10~{74$Q*Sfao3*}H4Y=Hd1gnP{TB zYChRqV*|eU%9SgKi5t=v;*cl^NLT3s48O(~#go8c1%d;>OKFf7M#N=`8WhY(xAOqR z3yRdrUzrrNln49uM%LZyk82xE1xhG*)DCeGf-*jaTpN)Pt*2iyZ2iO6^i`qa6;o9E zy0!MDfyEzHCHSMChV1V;8?0CA=$Ar^79fnEdx!>0BJfazmftuWvJDo0WIiIB7?Oo- zAH;vw1Zo9jD4?0oYy`+~cDN`&Y{b)GYedG=00N$D?k6^y$XCbHOs~GJ+IqD1siH#5 znS_=Kc9@>5+N}Z#T3@>i>&C<6j7sm4cTk&w?-!Btfwce`3WZ!bw1sYew{2TbAZ!J3 zIUtvaH`u>N6DPa=yk+sZ?PR$iBNK9k;kPb+pa0OGijl=(3>pgmfNMCwwQElS*GNge zDn#RCXL6=lVOG!3KhWViPizY2gB zS7O+aiWJcDf0KJ*2db8H2I1t=Id)t<5&wGg06L%R2mz|<7vr)Q_~$vvfp?%((DeW! zlqHqM%H%aH9Fv}Q2b^xPE6FQlc;mMcKZFAal?CsYFFb6pK6f)J5f-`i37tHAfT2Kc zK$6N?7gP;D-jE!iU8MGzcsHZG9@FkRFvrRA=PZ2QcE0PJxcH(_%d_OklPBLa;CJ}u zXCP~ogSvbFdte#_6I_~j#6y50M^|OJX87)+DM%V0TjQzj48r>OkUdfQdO5cA1u(4t zC~sY-3VvSmce7p$XXcq9IcWwc6a^U;&eWLF(D#vY<0ekx_?s0Pq0zq=OE3SQKjUAq z{+#;Vj34Ekt8C0D-8L5L>ViJ0b4u`ZLN;r9bG&V|bw-kL8t(@&1wv(L0^$DTQOo`n zxX>RY`3%qG*@8kcBT7rSFOqHDF?(rH)G`uVZM*H#!*kXqgKwEsok#D(6>&EJ_)M-) zR5S6hWwAZZByO>X|MK%hi+Ewls~&l>!tG-eT z-iNv$`!`OTWpVy^7e=v^ILUJ}=?c1Rtuy}h>uKKsI7>U1xtA4lUBkU4o@x;ggpmhSo2p0nW#|!ajQ%X1ib~AS9TU^zkk&tk44R4KV;Qd4X&>~1}#qxdH~C7Y1NZxy~o8PazCmGK_SAJVJ@uMM3@YcrR99|V}eYqN~1!f@0RmuO#X8O;iv{T}w zQGESc)P;uDCayO%;iosu-5$RM?^1vBqW0X_tzI8#Mkw&v#`K&CGFD3+yHsN8auoSF z#ylL^l``i{1Z&S{e(X3tr&6*1$77QRZ_Q*ZuB&aI$XS(#IBpD^4RC z|2~#i$(O)s#&7{1d*R0a51Cir5oaL(Mk1MpTgy$h2RDT&;H*H1>QE$3Zv6WLLLaGx z7q;9H>=WQT`Hd>y+4LK*ctqVwdy`!}O=Bg2IxNIxQTwtn-?epvtTdPA)>^5RJ8(@o zL;txZMFkMEfx=0v_oq^B-*B?n#5%jP7zu7WZdLuf-WCq-aCQnX6vyRv+R~WLHg1QH z#U>pRPkpYsbWI`n+m>7cM{gymQLx0IXCrKC3KbXu&^X;pvil-J_oY& z{o8_;p~~#zLtK>?75b|F`_=!prdu3!vBvsD7#@2XA3U|10m7t7tCGdbKTSuh~qvKN3YX9Fc=oO8CtTL3Sc}~d7vvCa1dOs ztGKo<5p=lD=e|R-9XI~0_MTQ_`pN_crClPnQ8IS^$)_V{;U2L<(hg}uNt;Oy5D};| z{;b}OjdTGMk{zBB+)87gk2p1t0BQztK3UWy4kgUgv*P?^mN)`P+_|2;< zsa7GZ6CxEZ7=WcD;DvN!Q)z!Bo`0|ND~Xo6U7O9#Dk%#32?4AU#&EizG6I&o1BX`b zS~XkBxo~)muDN85rDSva{-2WaR3EwSnt8I%`w%nA2C1v?_1WBx?84MjyVzB(V1wy0 zTUP>gHw0LVh>8kU@HC7H@01^7?jG!^k`2(34A2Tsi9Tze-`}!NBG=BqORCaPU zxBXWPj_LUxWoK@U4edrMG?vn}f-#7$a=YBQhv?mu7i~QnkE;r=X0hXHxdy){nig-w zdYfwciIQef<$S&qC;V<2_W!%aBY6q$5h#ZP?)xfo#;1?3jdg8**r`u#y^7=X=Vb)P z71V&e&SlClbV)?5xN^su9x=gJ+pvj!X072^7krgVdUrvIWZ@Or;)J8z;nzuPT?lB+Y5oBQ%$Ws;rNid{t5Sd_9}s+gG% zHX0-3lp9kL|Ice#{=Si88a{$t=j!@ip0hZB?ud|y{c>xYQmtLmIh^UwN^zMSQP zbAN&KHEx_W^)%_To4sQ{qW|yetfOu7?DcMJYU^L_+V!FEai`D^-;|HuYtv7Nu{@<@ z0tp8iGtEj(&-66jDfa(SqU91tyVNYT`dM(b%*8*up=m?ssRVcHPK@%|m#dFssYNSF z4xMn|>r~ZVSN-QWH|X!H%tfv=UW*QkMXh}Zylfr(Di36;SS8X&t>ouuw^l~3F;Ixz ze$!fexnRQ1O$62q_3XC>=}XY@(I>i$6 zLxGyV?s~mgN2ijv0Q|b}cN4Oi%N^%4gv&j9)V?(m+m$kQNedO2X;iz?leL+UGYxjm z14s@pK5xyYSem*$`6W345xuGdkkc!ffb-Q8gbVa@F}{%EU+D^F$Mw!sg{ZsdV_PGlYG zx#|lxjk9F2-HSht^eC9Am9<&bC^ys#qtDC&>I^d*h4Z+8F_q%}<%WDTUd2hzRzLD) zW$eEiWqLZx-%Be7ZnJV^L&lzsG&LXMS?92X19xM?)hb9*#nzQs95csVDO z>Em8m=1*vP1nOdsO`GX?;h*79DWOuBLrMi2&T6cA%q6nDsdDVbYcbXfC5v2Ip( zao*msaS)!wjNwHIYIv8}^b9>oLQ&wXN&JvI=v}N}%B&&yn^^VLt9_cUPnQM@E?9M> z{c0<0cf6T8Eb*08Eg_4WI1uRpk{T!UsRh5B1<}rxTn~NX0WHdcH?$~RSt{-#zp#nB zYbkY|kJ94_u3fr6eY1BjdR@!AgkubI&MK`qG~1-PuxLd;qSoHPB>fy+Pk z{6`8yF5LK)jI%La7-Ga*{Z0t^A{7-inva!>-RWj;w|;wFknnDib5@BmxmsW*gT5UT zX_&fST%h_5d2J^50~IH05_E_5@2w}_aC_Ke7Zg@*D-T&706f?krocGes7{Jq;s3Xq z207zW18|r*?b7jVB<4*S_j$yj!&tlU=?I$DF$cwer@( zzwG^$wBJ0ITGWppo;-Nj^bH&J8LC`WsB-O1 z!L|}wXL@OGJ_NaNnSCkgA!OY}NXj76>v{%`=_MdBh2K2{UcXP>4JY`6NrZ{iUgUq& zprmg?3+47z?h>cS4P?*HhWc-!c{shY=n<3 z1Y6QtB7KCN>@vU~5NNOE=wnmIDZcIQ0U#52W@e@uskIV; zLXMAi!O;#y7L}kQ7=pJRzPP8b?K}Tm#@Ba-$#mc&N!=r%t-zd)hd>QLtYiqt5WgS+ zX@IIvZ6j%GAa8K$2Q;w_mh(utRcB?1WKrdcKD9EV%io>G48=;EZ+bSpEOZ=yXjDsi z<7fOHt8&+^PTr4`E00YVJhAp6=4W}(yXp8=P6 zHQ-?L+Kq9Z$xlVXGezKeoyB7&*`vWu4GuK|5rSPTD%LVWxxAfyYQ9hQyurSop^{0v zRJ*Z#gF*R?Vhz~a$H(_mOKZc-1yK=n5%E0xNd!R@l>WaCPPjLM7O6ZN*7$SRum4P8 z`0J0nB9M}OZ#V)GMO?df4PrJOC;k9k#!J_JPIn{?Jr-h-z;W!)O$9i_+`AiCWqh4n zKc$#xWWQ(0CDZuYo(D*c!$(#ofyDfSnVSbLuTndHyJHl0jF6EM47=1FsepyRW;^q> zw~{eI`l8=tO8J9M0q#nI3q4hSez14x61t$(y3UTh$TwC-zX-bR?jZ)&|Ee77c>GCl zw2L9-^Bx#tmGfzGRs^t{jq3ZCI|}|_F4V@Y-4jqijBGiG8o=Q=?G!ad6&K3giN8G1>VepFM}fr)o(V z#YoRz-z5i=#yVylOf%dD?PkkLnBV8lb2P;ET`(T$dG~Sn===3^FYbW@N2h*TMNjLf zHkKt$g=!X1n6cWZu-5SKec@8sw{+Jv3C&5~#n6P>$afkh)}Cjx)!vU$TbFg#*okWVON?9fL|+rAKa<7`dj}x+fL(y%xvzv zzD!4lUt7MUw(lNTb431WzHo(q4R*mnN=2_XG|JjeKDSJEn)be!1g(L|=YN6(>e;-J z3N30pmbjj?a6BxSNd-N0r1r6vBzi6;F%)WWRfihkqO}*Jt4hAU5{{n>LJ%ciLT=h# zA+yJv1iNz-A<=qE+0WO6y^-sXM-lXeH$2Jvf-e74aQJTT>^%Q0#Ddcxrf~EveJZ#V zRIM{COk`2*DJFtVx5GejWd+hKEQ|)^uYwHqeJ~adcf;*_xyz2r9yG8zswU^5Rjx5s z>}%yr^~2}=N1IGuH7IF^7i|WOv9ryin-sZxId@xT{HM)b`MEz$R+(V)gk@pm=YB0J z>BWjZj!|s(fPavl}T)E4Kw>DIioO!4l^6My}a@fM(g_AJUVsb}A#c1k*B7pYc zkh??wKM87Phlo|ltw0v z%8gYeRflAGRAeevC@cL;@@`or;ioJ8*FbU@4RT`_GaVgDME=wn1R|5J#qLrUOzfsx`>EqQS)FP;YC~#5P4Fb zz#2YNzDdN@;4rArN2#Z@w$r_K&uMkAr{A&M2PR=g%EHol51~HC?>4}MXE%@<0uCLg zp>7Z|A@lH>P7l{?y`M@9C=(V6{WscpQ$g1qB=t{0aq)#!7I<0~zi$+sK_J8G2DYm7 z$OpD=kYoc12j-amWhPKjA``FhX=Zqk1%t%Y0MV0mw7W6O2s|Oyzi3b-+>C^R?OP-s z=CrwPn(zwAR>^yc;f#b7Wft@c@Yf*e>XBYhdN6_FjoR%D3c^KD?1;Z(gaCLL8g^?$ zgRn6DRXF5HUV@}Ux*Wbr>8?_0( z+2sDekR`y3{1(#y|UEIP9U4+Y0KcQ}C5fLtX3G4SJ2{8c*3E7>K zHEZyc*U?E>Fh9co+(;XB{-PM6MdR!b+|Fk&gN4*D%&l z(&Ibs$p$R?Y_LvGxoZnA!PgQNHtE){EVqyygXS?WFQozwC$0Un)@JFr*dp`e8>cLG z^m58>innIQ2l#o(YX;qKuPr1b=h>zkC---LHm%evbI7;#6FiR@s19;0oPDR>n2^p@ zpJS%ciRYoFarZ9yg@coYS39qSg-OGH?fT2I$ZUFiOtX>^4v_f0Gf6lnw*b4{v!Ihb zgB+-|t)8-6HwNoO-;P{nQc^B>j~qY<4!Ye`IQieeQY!#^h3x&b{qJBSn*whL2<(%R zk~lyO30|e2$tGxRP)^b?Fr*icgB8!{UVx{ZTyRgrbI_QNKz)BjjmM<+-PR$5iq=5( zmCM&ABM1*L0NaS(|5esLV3YLxYNpnIc17C1r|?2T51C$xuS( zDMJX?%rP}dkvUF=g9e#0ghM*f`)v33`{(`Z{qObhxn1Y%v-dvxJFI6tYprJ?Lj&sX zGWCsxi;dC;6zb)?JJ56kna~aVV#u9Cx$pvGSlGP0Rfzx^DR+EvZDg)4 zD%Duk-pJ_IMe{T%&*?$ou&K7kf#QkYC|NYCCc8xqZzYkoQ0^TcbiYy7ZC{Ar-ig;Y zz}DDj#>wlIW5uVz?2SPJOsbS9l|F-ubYirW7CC@syHMUzg2y2ocIEw(2Qbq~6F&x_ z?t?@b%XU=ZXpbMSB+7l~kQC8>?HVd3YY8#7xTEe!c?y{^&{a-{H1LV=_t{254j=wG z0*Aa4et1>rhCIELTga$Lg`tEjD}PIWEq0S}lpsnp0Hf+fea?hSLRiUBG`I_284~!U zSNp3ftYo#3rD5CiRmHytDQ|NS#fgnNRG|+bCFAIdgc+?vI`N0+X;6(Jo_b5YN!A&4 z7?w^L6g~|MPa8y`ihRuB6Vj2q$n~%o#n~aPtb0tT&*Zz;-E8%J8L5Nedup5m$OCsM zSR5rd7I49$DTxj1_{|~g*@~N@BX605DF;d3Jy0~rJGiBiOwUGp^eGeBuh3P?2Fl}< zvtSWTUB`$SfzNQ#ioMKkAL)u z+9#r5vqKzh*)J!$H92-KJTL%SD}#nMQ$ zV})M+_$>WfcgOx~=(BScn~Od=gSgi7cz7z=p!7!J#xKvB#99<}#IEu{M$;TRfe^5T zce^iM)4}m?i;^ZNuOjpFhIA&kC;6g^?M8`~cbr;o@|feoLd(+ihdZczrw)uFbx0-5 z4=AWmsf;w}^ue9SZ1Li~Gvq>b0eY=sd`cuKMbS@?=kiEt?*z++4f1#Slf z?6P>bx_ovW+Xg^ZLzG2n^`fn|)h6Q;lw zn241_;?;gnG1I~WRMl~q0!=L~sRo?qo%@wgTpT(>=kkSMWFH8b?b!Wi zrmH1a0oQr$lEb)UMX9P{=tT3rq_6rCl`E%nY-@DU_%cMpT{v3J^$YiQ6wwDMHku$FSMacp39IreE>0$#v570)^^kd+F4vzen$_ zd_^lkUTQ&>l9m=_=nN;rh)(?N8laQT?2m^yl4jzAHHi_c8r$7%`*eyBn_1A9V9uYsNnBSRu z&IG$&lgn=;b8Fk3+^yhx3UV*aoh{H%E;?yDSC3W@4%r?MZped45f1a1GMjYJkXWFo zmRaTXoqMumAxW$5RIO~~VPi`Jet7HyMYWE1%cMcw5gS`RwGq*X!HK5<=C!*zRuwdFazUf)FSFvva7Hq7^p##fLi=9~R8C1uctgX1-eZ@~ z(db^mz;;*y6oOTH8dX+r7AQ6Vvac^bFr~F#vnp(G>-x^+jGRq_?g2+CwlbOA)4rqa zd>vX7dDzM_u*Y%?p-ysDQfKdyO-R1!xUX`vyj`W(YFrPA@q@#pJyzk7I{0yTHHvG;H7Mw%VjXP$Ny(6WT3|~|jrjr-e=L#oiMI5pN59zH)aNUlTPu=; z%F_&m=Wc&WXJV#}At2F)`ovE70!b|%n2{w;1C~=&vpO|Yx8-X|ZyE`4}}3Re+~%L-!;@F`>6_-)0o5iqI;QGf9JgQ2)qXv!2GxUM?vi@#UJ_ zTu&-3Lu|NEVvSSsh>N?V$7Ut0hUrB)s#{_8?WCvZYVh8$IlFNlPv+_O&-HfLCNy<) zx=@%m2ahM)XK{wNUV(d42)BIu=Ze{OBuXwRG%E{}&$$a7azw7N!6KgEo3_t{U`|KK zJ*2VgLjyhB)wf%gTJj1;;t8at*cygGNZ(OWJ+yoyag3UuG-2J>lERQ*OCr^o+^vdC zjENBle7QI9<;MasWn~NFoX8D)H}}CecISk`4q=1s0pJo_Vv>>Cngeh7Isa~Xm*ZHr z&CaJ99A94_C3_4aSuqRvvEj`dy`x8OVK076STDyBUnCsjH0NvL9~0;hLOR{mVbasAQ!V-9){o5wo$C=e8eMR(^Qm2jv>#yYjJisWCH2PZ{1fFp(!OSwW#iE zS2RB2`dtGWPMc9`-~EX?kc3XjkGa{|GoUc=<;Pl<6s~6>maDyAN{gwDGbVi|F;}q$~06-m)bb?cwWQzs^R|XYAn%w+fuFzb&#;_(s8%7y$12@)p4^8B}2Z zs`wc*>EAlQ?qNdAabTA#S_-8wK_NVf#&8H@PRg|q&=3lYHA<>wk%C5Nzzs#h6VLz# z41?=Faean@A&4={MIVNxpIpaA*52QFK~YlD;O@!hxOwR;{m^Mvw!&@Q)qNb4e9P*w zg5UT?n!38OlxT2(M~@xT-kyDF7%Bdek=Pn0#06q20c^>^dnd7_=qEw(S_efV?}6p! zPLzSW|fp}YqId6^9*!)l}3p}hh79mp9c0SJg;nh`P!Gr zr>;hBvC4QTgiB)^mV<|zLQrU9he>C-U{QYU2nY-D>V5*%cJb711tyn{grL0Zv6jy@ zHVe>N&jAg>G4vu{+;AnU2szy|b3es4dL-;<~ zC@V9oy-_XxseXpatpUp;BAgU$6jAYTiXa0NE$Sl^vIF#O0QZQQQcFg4lh+V zIOwh+`Q`sJ(bFg?BGU9Zy0}=%KukB!m+7cSqOnTWe~>Gq4}`UJXwX{fylfYTPGcK@cU+3^nF}Psl!7kor25$L?p` zHZHCl?E2mL!yZ!tgiP@4X<}UeGdDSFmS~%jZM_Hm*ZrEbA^H`gj3NsnX}mjm;S5ff z^B0)q#)4e}c&}0!0IBgF;LbrR9mNW}@|FJUlZrmv{9z?Z$GWv^ zS3Kh;_QZ4WO7CZV>q$-A4umI77rr)gc~!}+Z2FS%`j>;Yr!ie~rsN@YS(bnzxT`tCmph@WUqY;}C!jRMBUf2%P zUcEZ*HrZ#2c6pXz&^v5HEm{_~OEB378>8lc&>?i%r>xDM7`TK7dK3PG=*WLT>E(I#P8aHcRMG-)CGivwXph);XuN> zrW&)8_{LKKxAzmfXDGLPDkfDHRw7uJ2UgaOv;-AjGXiQORuB;$o{AfQ2{$dHxu2_P zQJ|qvFxRrWhd)WxuRnU1h=70wB=*i|Y6iilM%n`Bkmu63_>rEHoC0oBKzCGuPEt9C z`Hr}_6qY&-~LwLGf8E6t%2+ z@*wFzTxiU^r!;6&V|f6gI zNF>=W*rH5;5rSs`GUNp0x9r}XkNfAiVSBLORBxFs z5db(;r<>PUMuX)Ow89YEIj=iU~-gfX76Nz0QkK2He@OgQ;-)} z;fyv92UGfO_RW`UCf&ad9lF={G-5E&4Ca-__q{M_ST-^ZFav+a943+0)9NHWfnklr zoD2*-Z>GLjZCG8^^kEa}4HsA%RFf73&p`Sp6-A*sTI^ugQ(=lU!moBc&4MI-+(Q&s zY%u<0Bj$1E>b<&sn5WAgfkM*KM&WGS+`>7XR~6Qhf*yh$$pR<}&&z+A(7~5I(+JJH zYl#KS|KLzF^xomtEA%JyGXMsdW&B~O&Jr= zT1asGN(y?M%6$>_QVm>R0^wp`+eZ4*D}7C8o@tNkz_s-4TeiS67KBbYy2P308t-?7 zW;6a}Ub9ATkwhXTzcOE~7-+1pESr)}_oG&t&%(*^*+7$t(>`+KNa0!G6Lw{ie?uIY zc=RMqtq=dYZU1PV4BtvgIMQ3;6H>(7C+%vmaxyt9hD7nUJ+3=7nZH!!TrM$-jET8o zzj^)o?HsL)@mKAU9 z=*C{mOid-1{^ER%5Enx=yv5)4kqai&YIWJH(Ek0$@%VuV=_2O9H z1peZTFP~qzt3}VaQ%xn2ahk*bjCr%1Bx-FEARWr^@ zlT#`bZ@y9s%U< zO7Q3xz_R(qz&T+hgm~m#0v;02loa6jLjsv>78Nda{OACtN5p&Sa7Jyh%Yp+Mp3}*rLg=wN zYz@j>iT7b_Y{T0%K~IjxRV!f^2Wunga4#t649K_^mV*KUZ;Nw5)Uj;Z)J&zCrEBg; zkaIp7ezC+1z(*d|Ri^QC*ltzFtISS)t@y8!ycm_YbImQ1BK>YSX&pZ^WNAW!7*tAydYGS1P>o>{a{nneX-!|%on!`6|tLta#CiqZ@q147y{18 zaKV#UBJ(adDo?f$%T`VrTt9M2WgOxV&_FEPTZ8b_rQqXqEQ2cXmRb*WK8xm>530Nivz*R zFDlwx+~ii0gl6I*(Ds?SvxjuM#n{mD@1k4I4t1g|JAxS!7ZFCvuSaXZn1*Hd(-_WY z1LqL*m_tB)6j)~F!veU4F^I-D%@ly>hR(*@hv}|RQcVZ=?cb7|7BJaoczDsYqCD`D z-#H2`%sRvoH}Jy_Sqr?>SBS-ob!3LM)Gxu4wR|*Cy>)UNbQ@C&dldw)VzL!cl0y3Wnd`4zGAmY z?G4|Rahe`QD1%_LuAH;AC1|dT5e)ixCOjUar)S$fe+Na49y=C^e^dsChJIybnz_l> z;Z|CBH1B+@)Rk0B8ynN-eXbqamO{}0W4TtfDlLMKuxUQ_`TC9n^Jdo&qP4albySk} zV`m3L(++xV?)jWaQ+7eXb_+! zBcWM~5y;AABlQOpKZT__T@juU?yCd-z$JT#p+q3tjg0Yvtiv=ocm}KZrU@n|o=+D#P6Rs;-Y$3>g3z$9 z#4bdHLLi65)r(xJ66en+Omn+@{1S>?SIcJTodb;&NM4w9VCf@1#^qFXLBSES5!t3i z8T!j05Z}WoU&T;Igy=4DSOj9`oh5cAO$mxr&k1PMQ_+qg9o%sA>nK(90bWDQ$%|Kr zc+xW-aD6avt!(9dSzsJ^tJVB|eLtxkD+O!DcHwqU`w}!CMzDuV8yzML1uThDXh>KX z6@IJ}XZ;Lh#l|B|ECudU$>5Zwo%;Kv+Tsqtq0WY00uf%VKkf+1>?c(B9%GTKT*0NM z@87?l47FMzWo6ojul?g4wCgR(ndRkjkD=}eE0f`qhk3@{{3nIRamVDb4&1 z*^c;$0^$h}7Mb9w1Xdm^q89`ctDlE{Nbnqo+8QR{r=2G3E~H*V956x|T{FK3WnYL* zYqZlpMnOEF%3n=?6Rj@mmUARR$jH+f2W8NPn26Fm@?Pu^OXXB>ebVxKBW{=kSh^eM zLRjA&(V|mm?u!5(!7%5aN!N5nfh~RzWgvcLPLncM{^!3y^*^txjevz-In&C!{{gfA zLR_FvVw;DzwQ|gVPqm)dU EOSDependentVars: """Get an agglomerated array of the dependent variables.""" return EOSDependentVars( @@ -137,6 +143,40 @@ def dependent_vars(self, cv: ConservedVars) -> EOSDependentVars: ) +class MixtureEOS(GasEOS): + r"""Abstract interface to gas mixture equation of state class. + + This EOS base class extends the GasEOS base class to include the + necessary interface for dealing with gas mixtures. + + .. automethod:: get_density + .. automethod:: get_species_molecular_weights + .. automethod:: get_production_rates + .. automethod:: species_enthalpies + .. automethod:: get_species_source_terms + """ + + def get_density(self, pressure, temperature, species_mass_fractions): + """Get the density from pressure, temperature, and species fractions (Y).""" + raise NotImplementedError() + + def get_species_molecular_weights(self): + """Get the species molecular weights.""" + raise NotImplementedError() + + def species_enthalpies(self, cv: ConservedVars): + """Get the species specific enthalpies.""" + raise NotImplementedError() + + def get_production_rates(self, cv: ConservedVars): + """Get the production rate for each species.""" + raise NotImplementedError() + + def get_species_source_terms(self, cv: ConservedVars): + r"""Get the species mass source terms to be used on the RHS for chemistry.""" + raise NotImplementedError() + + class IdealSingleGas(GasEOS): r"""Ideal gas law single-component gas ($p = \rho{R}{T}$). @@ -379,7 +419,7 @@ def get_internal_energy(self, temperature, mass, species_mass_fractions=None): return self._gas_const * mass * temperature / (self._gamma - 1) -class PyrometheusMixture(GasEOS): +class PyrometheusMixture(MixtureEOS): r"""Ideal gas mixture ($p = \rho{R}_\mathtt{mix}{T}$). This is the :mod:`pyrometheus`-based EOS. Please refer to the :any:`documentation @@ -410,6 +450,7 @@ class PyrometheusMixture(GasEOS): .. automethod:: get_density .. automethod:: get_species_molecular_weights .. automethod:: get_production_rates + .. automethod:: species_enthalpies .. automethod:: get_species_source_terms """ @@ -603,6 +644,10 @@ def get_species_molecular_weights(self): """Get the species molecular weights.""" return self._pyrometheus_mech.wts + def species_enthalpies(self, cv: ConservedVars): + """Get the species specific enthalpies.""" + return self._pyrometheus_mech.get_species_enthalpies_rt(self.temperature(cv)) + def get_production_rates(self, cv: ConservedVars): r"""Get the production rate for each species. diff --git a/mirgecom/fluid.py b/mirgecom/fluid.py index 71e993cb4..9399df7db 100644 --- a/mirgecom/fluid.py +++ b/mirgecom/fluid.py @@ -243,6 +243,11 @@ def velocity(self): """Return the fluid velocity = momentum / mass.""" return self.momentum / self.mass + @property + def nspecies(self): + """Return the number of mixture species.""" + return len(self.species_mass) + @property def species_mass_fractions(self): """Return the species mass fractions y = species_mass / mass.""" diff --git a/mirgecom/flux.py b/mirgecom/flux.py index fa760154e..0b7f65ab9 100644 --- a/mirgecom/flux.py +++ b/mirgecom/flux.py @@ -3,10 +3,9 @@ Numerical Flux Routines ^^^^^^^^^^^^^^^^^^^^^^^ -.. autofunction:: lfr_flux -.. autofunction:: central_scalar_flux -.. autofunction:: central_vector_flux - +.. autofunction:: gradient_flux_central +.. autofunction:: divergence_flux_central +.. autofunction:: divergence_flux_lfr """ __copyright__ = """ @@ -40,24 +39,28 @@ ) -def central_scalar_flux(trace_pair, normal): - r"""Compute a central scalar flux. +def gradient_flux_central(u_tpair, normal): + r"""Compute a central scalar flux for the gradient operator. - The central scalar flux, $h$, is calculated as: + The central scalar flux, $\mathbf{h}$, of a scalar quantity $u$ is calculated as: .. math:: - h(\mathbf{u}^-, \mathbf{u}^+; \mathbf{n}) = \frac{1}{2} - \left(\mathbf{u}^{+}+\mathbf{u}^{-}\right)\hat{n} + \mathbf{h}({u}^-, {u}^+; \mathbf{n}) = \frac{1}{2} + \left({u}^{+}+{u}^{-}\right)\mathbf{\hat{n}} + + where ${u}^-, {u}^+$, are the scalar function values on the interior + and exterior of the face on which the central flux is to be calculated, and + $\mathbf{\hat{n}}$ is the *normal* vector. - where $\mathbf{u}^-, \mathbf{u}^+$, are the vector of independent scalar - components and scalar solution components on the interior and exterior of the - face on which the central flux is to be calculated, and $\hat{n}$ is the normal - vector. + *u_tpair* is the :class:`~grudge.trace_pair.TracePair` representing the scalar + quantities ${u}^-, {u}^+$. *u_tpair* may also represent a vector-quantity + :class:`~grudge.trace_pair.TracePair`, and in this case the central scalar flux + is computed on each component of the vector quantity as an independent scalar. Parameters ---------- - trace_pair: `grudge.trace_pair.TracePair` + u_tpair: `grudge.trace_pair.TracePair` Trace pair for the face upon which flux calculation is to be performed normal: numpy.ndarray object array of :class:`meshmode.dof_array.DOFArray` with outward-pointing @@ -69,28 +72,23 @@ def central_scalar_flux(trace_pair, normal): object array of `meshmode.dof_array.DOFArray` with the central scalar flux for each scalar component. """ - tp_avg = trace_pair.avg + tp_avg = u_tpair.avg + tp_join = tp_avg if isinstance(tp_avg, DOFArray): return tp_avg*normal elif isinstance(tp_avg, ConservedVars): tp_join = tp_avg.join() - elif isinstance(tp_avg, np.ndarray): - tp_join = tp_avg - - ncomp = len(tp_join) - if ncomp > 1: - result = np.empty((ncomp, len(normal)), dtype=object) - for i in range(ncomp): - result[i] = tp_join[i] * normal - else: - result = tp_join*normal + + result = np.outer(tp_join, normal) + if isinstance(tp_avg, ConservedVars): return make_conserved(tp_avg.dim, q=result) - return result + else: + return result -def central_vector_flux(trace_pair, normal): - r"""Compute a central vector flux. +def divergence_flux_central(trace_pair, normal): + r"""Compute a central flux for the divergence operator. The central vector flux, $h$, is calculated as: @@ -120,7 +118,7 @@ def central_vector_flux(trace_pair, normal): return trace_pair.avg@normal -def lfr_flux(cv_tpair, f_tpair, normal, lam): +def divergence_flux_lfr(cv_tpair, f_tpair, normal, lam): r"""Compute Lax-Friedrichs/Rusanov flux after [Hesthaven_2008]_, Section 6.6. The Lax-Friedrichs/Rusanov flux is calculated as: diff --git a/mirgecom/initializers.py b/mirgecom/initializers.py index 8ad1d89d2..b41f6e7ab 100644 --- a/mirgecom/initializers.py +++ b/mirgecom/initializers.py @@ -13,6 +13,7 @@ .. automethod: make_pulse .. autoclass:: MixtureInitializer .. autoclass:: PlanarDiscontinuity +.. autoclass:: PlanarPoiseuille """ __copyright__ = """ @@ -965,7 +966,6 @@ def __call__(self, x_vec, eos, **kwargs): mom = mass * velocity internal_energy = eos.get_internal_energy(temperature=temperature, species_mass_fractions=y) - kinetic_energy = 0.5 * np.dot(velocity, velocity) energy = mass * (internal_energy + kinetic_energy) @@ -1054,8 +1054,7 @@ def __init__( self._xdir = self._dim - 1 def __call__(self, x_vec, eos, *, time=0.0): - """ - Create the mixture state at locations *x_vec*. + """Create the mixture state at locations *x_vec*. Parameters ---------- @@ -1106,3 +1105,116 @@ def __call__(self, x_vec, eos, *, time=0.0): return make_conserved(dim=self._dim, mass=mass, energy=energy, momentum=mom, species_mass=specmass) + + +class PlanarPoiseuille: + r"""Initializer for the planar Poiseuille case. + + The planar Poiseuille case is defined as a viscous flow between two + stationary parallel sides with a uniform pressure drop prescribed + as *p_hi* at the inlet and *p_low* at the outlet. See the figure below: + + .. figure:: ../figures/poiseuille.png + :scale: 50 % + :alt: Poiseuille domain illustration + + Illustration of the Poiseuille case setup + + The exact Poiseuille solution is defined by the following: + $$ + P(x) &= P_{\text{hi}} + P'x\\ + v_x &= \frac{-P'}{2\mu}y(h-y), v_y = 0\\ + \rho &= \rho_0\\ + \rho{E} &= \frac{P(x)}{(\gamma-1)} + \frac{\rho}{2}(\mathbf{v}\cdot\mathbf{v}) + $$ + + Here, $P'$ is the constant slope of the linear pressure gradient from the inlet + to the outlet and is calculated as: + $$ + P' = \frac{(P_{\text{low}}-P_{\text{hi}})}{\text{length}} + $$ + $v_x$, and $v_y$ are respectively the x and y components of the velocity, + $\mathbf{v}$, and $\rho_0$ is the supplied constant density of the fluid. + """ + + def __init__(self, p_hi=100100., p_low=100000., mu=1.0, height=.02, length=.1, + density=1.0): + """Initialize the Poiseuille solution initializer. + + Parameters + ---------- + p_hi: float + Pressure at the inlet (default=100100) + p_low: float + Pressure at the outlet (default=100000) + mu: float + Fluid viscosity, (default = 1.0) + height: float + Height of the domain, (default = .02) + length: float + Length of the domain, (default = .1) + density: float + Constant density of the fluid, (default=1.0) + """ + self.length = length + self.height = height + self.dpdx = (p_low - p_hi)/length + self.rho = density + self.mu = mu + self.p_hi = p_hi + + def __call__(self, x_vec, eos, cv=None, **kwargs): + r"""Create the Poiseuille solution. + + Parameters + ---------- + x_vec: numpy.ndarray + Array of :class:`~meshmode.dof_array.DOFArray` representing the 2D + coordinates of the points at which the solution is desired + eos: :class:`~mirgecom.eos.GasEOS` + A gas equation of state + cv: :class:`~mirgecom.fluid.ConservedVars` + Optional fluid state to supply fluid density and velocity if needed. + + Returns + ------- + :class:`~mirgecom.fluid.ConservedVars` + The Poiseuille solution state + """ + x = x_vec[0] + y = x_vec[1] + p_x = self.p_hi + self.dpdx*x + + if cv is not None: + mass = cv.mass + velocity = cv.velocity + else: + mass = self.rho*x/x + u_x = -self.dpdx*y*(self.height - y)/(2*self.mu) + velocity = make_obj_array([u_x, 0*x]) + + ke = .5*np.dot(velocity, velocity)*mass + rho_e = p_x/(eos.gamma(cv)-1) + ke + return make_conserved(2, mass=mass, energy=rho_e, + momentum=mass*velocity) + + def exact_grad(self, x_vec, eos, cv_exact): + """Return the exact gradient of the Poiseuille state.""" + y = x_vec[1] + x = x_vec[0] + ones = x / x + mass = cv_exact.mass + velocity = cv_exact.velocity + dvxdy = -self.dpdx*(self.height-2*y)/(2*self.mu) + dvdy = make_obj_array([dvxdy, 0*x]) + dedx = self.dpdx/(eos.gamma(cv_exact)-1)*ones + dedy = mass*np.dot(velocity, dvdy) + dmass = make_obj_array([0*x, 0*x]) + denergy = make_obj_array([dedx, dedy]) + dvx = make_obj_array([0*x, dvxdy]) + dvy = make_obj_array([0*x, 0*x]) + dv = np.stack((dvx, dvy)) + dmom = mass*dv + species_mass = velocity*cv_exact.species_mass.reshape(-1, 1) + return make_conserved(2, mass=dmass, energy=denergy, + momentum=dmom, species_mass=species_mass) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index 2034dcfef..79070e355 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -41,7 +41,7 @@ from meshmode.dof_array import thaw from mirgecom.fluid import compute_wavespeed from grudge.trace_pair import TracePair -from mirgecom.flux import lfr_flux +from mirgecom.flux import divergence_flux_lfr from mirgecom.fluid import make_conserved @@ -103,7 +103,7 @@ def inviscid_facial_flux(discr, eos, cv_tpair, local=False): normal = thaw(actx, discr.normal(cv_tpair.dd)) # todo: user-supplied flux routine - flux_weak = lfr_flux(cv_tpair, flux_tpair, normal=normal, lam=lam) + flux_weak = divergence_flux_lfr(cv_tpair, flux_tpair, normal=normal, lam=lam) if local is False: return discr.project(cv_tpair.dd, "all_faces", flux_weak) diff --git a/mirgecom/navierstokes.py b/mirgecom/navierstokes.py index db02c56d5..33dde7406 100644 --- a/mirgecom/navierstokes.py +++ b/mirgecom/navierstokes.py @@ -70,7 +70,7 @@ viscous_facial_flux ) from mirgecom.flux import ( - central_scalar_flux + gradient_flux_central ) from mirgecom.fluid import make_conserved from mirgecom.operators import ( @@ -127,7 +127,7 @@ def _elbnd_flux(discr, compute_interior_flux, compute_boundary_flux, def scalar_flux_interior(int_tpair): normal = thaw(actx, discr.normal(int_tpair.dd)) # Hard-coding central per [Bassi_1997]_ eqn 13 - flux_weak = central_scalar_flux(int_tpair, normal) + flux_weak = gradient_flux_central(int_tpair, normal) return discr.project(int_tpair.dd, "all_faces", flux_weak) def get_q_flux_bnd(btag): diff --git a/mirgecom/operators.py b/mirgecom/operators.py index 407c205eb..72d9515c1 100644 --- a/mirgecom/operators.py +++ b/mirgecom/operators.py @@ -58,9 +58,9 @@ def div_operator(discr, u, flux): ---------- discr: grudge.eager.EagerDGDiscretization the discretization to use - u: np.ndarray + u: numpy.ndarray the vector-valued function for which divergence is to be calculated - flux: np.ndarray + flux: numpy.ndarray the boundary fluxes across the faces of the element Returns ------- diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index c9041dc77..5a792ae76 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -103,9 +103,8 @@ def get_sim_timestep(discr, state, t, dt, cfl, eos, cfl: float The current CFL number eos: :class:`~mirgecom.eos.GasEOS` - Gas equation-of-state supporting speed_of_sound, and, optionally for viscous - fluids, a non-empty :class:`~mirgecom.transport.TransportModel` for viscous - transport properties. + Gas equation-of-state optionally with a non-empty + :class:`~mirgecom.transport.TransportModel` for viscous transport properties. constant_cfl: bool True if running constant CFL mode diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 7906608ef..5112f826e 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -5,7 +5,7 @@ """ __copyright__ = """ -Copyright (C) 2021 University of Illinois Board of Trustees +Copyright (C) 2020-21 University of Illinois Board of Trustees """ __license__ = """ diff --git a/mirgecom/viscous.py b/mirgecom/viscous.py index e774a5d3c..04f6233bc 100644 --- a/mirgecom/viscous.py +++ b/mirgecom/viscous.py @@ -43,7 +43,6 @@ import numpy as np from pytools import memoize_in -from pytools.obj_array import make_obj_array from mirgecom.fluid import ( velocity_gradient, species_mass_fraction_gradient, @@ -51,6 +50,7 @@ ) from meshmode.dof_array import thaw, DOFArray import arraycontext +from mirgecom.eos import MixtureEOS def viscous_stress_tensor(discr, eos, cv, grad_cv): @@ -66,6 +66,8 @@ def viscous_stress_tensor(discr, eos, cv, grad_cv): Parameters ---------- + discr: :class:`grudge.eager.EagerDGDiscretization` + The discretization to use eos: :class:`~mirgecom.eos.GasEOS` A gas equation of state with a non-empty :class:`~mirgecom.transport.TransportModel`. @@ -94,7 +96,7 @@ def viscous_stress_tensor(discr, eos, cv, grad_cv): def diffusive_flux(discr, eos, cv, grad_cv): r"""Compute the species diffusive flux vector, ($\mathbf{J}_{\alpha}$). - The species diffussive flux is defined by: + The species diffusive flux is defined by: .. math:: @@ -106,6 +108,8 @@ def diffusive_flux(discr, eos, cv, grad_cv): Parameters ---------- + discr: :class:`grudge.eager.EagerDGDiscretization` + The discretization to use eos: :class:`~mirgecom.eos.GasEOS` A gas equation of state with a non-empty :class:`~mirgecom.transport.TransportModel` @@ -119,19 +123,11 @@ def diffusive_flux(discr, eos, cv, grad_cv): numpy.ndarray The species diffusive flux vector, $\mathbf{J}_{\alpha}$ """ - nspecies = len(cv.species_mass) transport = eos.transport_model() grad_y = species_mass_fraction_gradient(discr, cv, grad_cv) d = transport.species_diffusivity(eos, cv) - - # TODO: Better way? - obj_ary = -make_obj_array([cv.mass*d[i]*grad_y[i] for i in range(nspecies)]) - diffusive_flux = np.empty(shape=(nspecies, discr.dim), dtype=object) - for idx, v in enumerate(obj_ary): - diffusive_flux[idx] = v - - return diffusive_flux + return -cv.mass*d.reshape(-1, 1)*grad_y def conductive_heat_flux(discr, eos, cv, grad_t): @@ -141,12 +137,14 @@ def conductive_heat_flux(discr, eos, cv, grad_t): .. math:: - \mathbf{q}_{c} = \kappa\nabla{T}, + \mathbf{q}_{c} = -\kappa\nabla{T}, with thermal conductivity $\kappa$, and gas temperature $T$. Parameters ---------- + discr: :class:`grudge.eager.EagerDGDiscretization` + The discretization to use eos: :class:`~mirgecom.eos.GasEOS` A gas equation of state with a non-empty :class:`~mirgecom.transport.TransportModel` @@ -161,7 +159,7 @@ def conductive_heat_flux(discr, eos, cv, grad_t): The conductive heat flux vector """ transport = eos.transport_model() - return transport.thermal_conductivity(eos, cv)*grad_t + return -transport.thermal_conductivity(eos, cv)*grad_t def diffusive_heat_flux(discr, eos, cv, j): @@ -185,6 +183,8 @@ def diffusive_heat_flux(discr, eos, cv, j): Parameters ---------- + discr: :class:`grudge.eager.EagerDGDiscretization` + The discretization to use eos: mirgecom.eos.GasEOS A gas equation of state with a non-empty :class:`~mirgecom.transport.TransportModel` @@ -196,12 +196,12 @@ def diffusive_heat_flux(discr, eos, cv, j): Returns ------- numpy.ndarray - The total diffusive heath flux vector + The total diffusive heat flux vector """ - numspecies = len(cv.species_mass) - transport = eos.transport_model() - d = transport.species_diffusivity(eos, cv) - return sum(d[i]*j[i] for i in range(numspecies)) + if isinstance(eos, MixtureEOS): + h_alpha = eos.species_enthalpies(cv) + return sum(h_alpha.reshape(-1, 1) * j) + return 0 # TODO: We could easily make this more general (dv, grad_dv) @@ -215,11 +215,11 @@ def viscous_flux(discr, eos, cv, grad_cv, grad_t): .. math:: \mathbf{F}_V = [0,\tau\cdot\mathbf{v} - \mathbf{q}, - \tau_{:i},-\mathbf{J}_\alpha], + \tau,-\mathbf{J}_\alpha], with fluid velocity ($\mathbf{v}$), viscous stress tensor - ($\mathbf{\tau}$), and diffusive flux for each species - ($\mathbf{J}_\alpha$). + ($\mathbf{\tau}$), heat flux (\mathbf{q}), and diffusive flux + for each species ($\mathbf{J}_\alpha$). .. note:: @@ -230,9 +230,10 @@ def viscous_flux(discr, eos, cv, grad_cv, grad_t): Parameters ---------- + discr: :class:`grudge.eager.EagerDGDiscretization` + The discretization to use eos: :class:`~mirgecom.eos.GasEOS` - A gas equation of state with a non-empty - :class:`~mirgecom.transport.TransportModel` + A gas equation of state cv: :class:`~mirgecom.fluid.ConservedVars` Fluid state grad_cv: :class:`~mirgecom.fluid.ConservedVars` @@ -242,23 +243,26 @@ def viscous_flux(discr, eos, cv, grad_cv, grad_t): Returns ------- - :class:`~mirgecom.fluid.ConservedVars` - The viscous transport flux vector + :class:`~mirgecom.fluid.ConservedVars` or float + The viscous transport flux vector if viscous transport properties + are provided, scalar zero otherwise. """ + transport = eos.transport_model() + if transport is None: + return 0 + dim = cv.dim + viscous_mass_flux = 0 * cv.momentum j = diffusive_flux(discr, eos, cv, grad_cv) - heat_flux = conductive_heat_flux(discr, eos, cv, grad_t) - # heat_flux = (conductive_heat_flux(discr, eos, q, grad_t) - # + diffusive_heat_flux(discr, eos, q, j)) - tau = viscous_stress_tensor(discr, eos, cv, grad_cv) - viscous_mass_flux = 0 * cv.momentum - viscous_energy_flux = np.dot(tau, cv.velocity) - heat_flux + heat_flux_diffusive = diffusive_heat_flux(discr, eos, cv, j) - # passes the right (empty) shape for diffusive flux when no species - # TODO: fix single gas join_conserved for vectors at each cons eqn - if len(j) == 0: - j = cv.momentum * cv.species_mass.reshape(-1, 1) + tau = viscous_stress_tensor(discr, eos, cv, grad_cv) + viscous_energy_flux = ( + np.dot(tau, cv.velocity) + - conductive_heat_flux(discr, eos, cv, grad_t) + - heat_flux_diffusive + ) return make_conserved(dim, mass=viscous_mass_flux, @@ -272,9 +276,10 @@ def viscous_facial_flux(discr, eos, cv_tpair, grad_cv_tpair, grad_t_tpair, Parameters ---------- + discr: :class:`grudge.eager.EagerDGDiscretization` + The discretization to use eos: :class:`~mirgecom.eos.GasEOS` - Implementing the pressure and temperature functions for - returning pressure and temperature as a function of the state q. + A gas equation of state cv_tpair: :class:`grudge.trace_pair.TracePair` Trace pair of :class:`~mirgecom.fluid.ConservedVars` with the fluid solution on the faces @@ -286,7 +291,7 @@ def viscous_facial_flux(discr, eos, cv_tpair, grad_cv_tpair, grad_t_tpair, local: bool Indicates whether to skip projection of fluxes to "all_faces" or not. If set to *False* (the default), the returned fluxes are projected to - "all_faces." If set to *True*, the returned fluxes are not projected to + "all_faces". If set to *True*, the returned fluxes are not projected to "all_faces"; remaining instead on the boundary restriction. Returns @@ -303,12 +308,11 @@ def viscous_facial_flux(discr, eos, cv_tpair, grad_cv_tpair, grad_t_tpair, grad_t_tpair.int) f_ext = viscous_flux(discr, eos, cv_tpair.ext, grad_cv_tpair.ext, grad_t_tpair.ext) - # This bit hard-codes BR1 central flux, use mirgecom.flux.central? f_avg = 0.5*(f_int + f_ext) flux_weak = make_conserved(cv_tpair.int.dim, q=(f_avg.join() @ normal)) - if local is False: + if not local: return discr.project(cv_tpair.dd, "all_faces", flux_weak) return flux_weak @@ -321,7 +325,7 @@ def get_viscous_timestep(discr, eos, cv): discr: grudge.eager.EagerDGDiscretization the discretization to use eos: :class:`~mirgecom.eos.GasEOS` - An equation of state implementing the speed_of_sound method + A gas equation of state cv: :class:`~mirgecom.fluid.ConservedVars` Fluid solution @@ -361,8 +365,7 @@ def get_viscous_cfl(discr, eos, dt, cv): discr: :class:`grudge.eager.EagerDGDiscretization` the discretization to use eos: :class:`~mirgecom.eos.GasEOS` - Implementing the pressure and temperature functions for - returning pressure and temperature as a function of the state q. + A gas equation of state dt: float or :class:`~meshmode.dof_array.DOFArray` A constant scalar dt or node-local dt cv: :class:`~mirgecom.fluid.ConservedVars` diff --git a/test/test_av.py b/test/test_av.py index 626893753..858afd7b4 100644 --- a/test/test_av.py +++ b/test/test_av.py @@ -183,8 +183,8 @@ def soln_gradient_flux(self, disc, btag, soln, **kwargs): interior=soln_int, exterior=soln_int) nhat = thaw(actx, disc.normal(btag)) - from mirgecom.flux import central_scalar_flux - flux_weak = central_scalar_flux(bnd_pair, normal=nhat) + from mirgecom.flux import gradient_flux_central + flux_weak = gradient_flux_central(bnd_pair, normal=nhat) return disc.project(btag, "all_faces", flux_weak) def av_flux(self, disc, btag, diffusion, **kwargs): @@ -194,8 +194,8 @@ def av_flux(self, disc, btag, diffusion, **kwargs): from grudge.trace_pair import TracePair bnd_grad_pair = TracePair(btag, interior=grad_soln_minus, exterior=grad_soln_plus) - from mirgecom.flux import central_vector_flux - flux_weak = central_vector_flux(bnd_grad_pair, normal=nhat) + from mirgecom.flux import divergence_flux_central + flux_weak = divergence_flux_central(bnd_grad_pair, normal=nhat) return disc.project(btag, "all_faces", flux_weak) boundaries = {BTAG_ALL: TestBoundary()} diff --git a/test/test_bc.py b/test/test_bc.py index 97862e997..bf3f9158d 100644 --- a/test/test_bc.py +++ b/test/test_bc.py @@ -227,12 +227,12 @@ def test_noslip(actx_factory, dim): print(f"{nhat=}") # h = 1.0 / np1 - from mirgecom.flux import central_scalar_flux + from mirgecom.flux import gradient_flux_central def scalar_flux_interior(int_tpair): normal = thaw(actx, discr.normal(int_tpair.dd)) # Hard-coding central per [Bassi_1997]_ eqn 13 - flux_weak = central_scalar_flux(int_tpair, normal) + flux_weak = gradient_flux_central(int_tpair, normal) return discr.project(int_tpair.dd, "all_faces", flux_weak) # utility to compare stuff on the boundary only @@ -347,12 +347,12 @@ def test_prescribedviscous(actx_factory, dim): nhat = thaw(actx, discr.normal(BTAG_ALL)) print(f"{nhat=}") - from mirgecom.flux import central_scalar_flux + from mirgecom.flux import gradient_flux_central def scalar_flux_interior(int_tpair): normal = thaw(actx, discr.normal(int_tpair.dd)) # Hard-coding central per [Bassi_1997]_ eqn 13 - flux_weak = central_scalar_flux(int_tpair, normal) + flux_weak = gradient_flux_central(int_tpair, normal) return discr.project(int_tpair.dd, "all_faces", flux_weak) # utility to compare stuff on the boundary only diff --git a/test/test_navierstokes.py b/test/test_navierstokes.py new file mode 100644 index 000000000..10ef64829 --- /dev/null +++ b/test/test_navierstokes.py @@ -0,0 +1,330 @@ +"""Test the Navier-Stokes gas dynamics module.""" + +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import numpy as np +import numpy.random +import numpy.linalg as la # noqa +import pyopencl.clmath # noqa +import logging +import pytest + +from pytools.obj_array import ( + flat_obj_array, + make_obj_array, +) + +from meshmode.dof_array import thaw +from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa +from mirgecom.navierstokes import ns_operator +from mirgecom.fluid import make_conserved +from grudge.dof_desc import DTAG_BOUNDARY + +from mirgecom.boundary import ( + DummyBoundary, + PrescribedViscousBoundary, + IsothermalNoSlipBoundary +) +from mirgecom.eos import IdealSingleGas +from mirgecom.transport import SimpleTransport +from grudge.eager import EagerDGDiscretization +from meshmode.array_context import ( # noqa + pytest_generate_tests_for_pyopencl_array_context + as pytest_generate_tests) + + +logger = logging.getLogger(__name__) + + +@pytest.mark.parametrize("nspecies", [0, 10]) +@pytest.mark.parametrize("dim", [1, 2, 3]) +@pytest.mark.parametrize("order", [1, 2, 3]) +def test_uniform_rhs(actx_factory, nspecies, dim, order): + """Test the Navier-Stokes operator using a trivial constant/uniform state. + + This state should yield rhs = 0 to FP. The test is performed for 1, 2, + and 3 dimensions, with orders 1, 2, and 3, with and without passive species. + """ + actx = actx_factory() + + tolerance = 1e-9 + + from pytools.convergence import EOCRecorder + eoc_rec0 = EOCRecorder() + eoc_rec1 = EOCRecorder() + # for nel_1d in [4, 8, 12]: + for nel_1d in [4, 8]: + from meshmode.mesh.generation import generate_regular_rect_mesh + mesh = generate_regular_rect_mesh( + a=(-0.5,) * dim, b=(0.5,) * dim, nelements_per_axis=(nel_1d,) * dim + ) + + logger.info( + f"Number of {dim}d elements: {mesh.nelements}" + ) + + discr = EagerDGDiscretization(actx, mesh, order=order) + zeros = discr.zeros(actx) + ones = zeros + 1.0 + + mass_input = discr.zeros(actx) + 1 + energy_input = discr.zeros(actx) + 2.5 + + mom_input = make_obj_array( + [discr.zeros(actx) for i in range(discr.dim)] + ) + + mass_frac_input = flat_obj_array( + [ones / ((i + 1) * 10) for i in range(nspecies)] + ) + species_mass_input = mass_input * mass_frac_input + num_equations = dim + 2 + len(species_mass_input) + + cv = make_conserved( + dim, mass=mass_input, energy=energy_input, momentum=mom_input, + species_mass=species_mass_input) + + expected_rhs = make_conserved( + dim, q=make_obj_array([discr.zeros(actx) + for i in range(num_equations)]) + ) + mu = 1.0 + kappa = 0.0 + spec_diffusivity = 0 * np.ones(nspecies) + transport_model = SimpleTransport(viscosity=mu, thermal_conductivity=kappa, + species_diffusivity=spec_diffusivity) + eos = IdealSingleGas(transport_model=transport_model) + boundaries = {BTAG_ALL: DummyBoundary()} + + ns_rhs = ns_operator(discr, eos=eos, boundaries=boundaries, cv=cv, t=0.0) + + rhs_resid = ns_rhs - expected_rhs + rho_resid = rhs_resid.mass + rhoe_resid = rhs_resid.energy + mom_resid = rhs_resid.momentum + rhoy_resid = rhs_resid.species_mass + + rho_rhs = ns_rhs.mass + rhoe_rhs = ns_rhs.energy + rhov_rhs = ns_rhs.momentum + rhoy_rhs = ns_rhs.species_mass + + logger.info( + f"rho_rhs = {rho_rhs}\n" + f"rhoe_rhs = {rhoe_rhs}\n" + f"rhov_rhs = {rhov_rhs}\n" + f"rhoy_rhs = {rhoy_rhs}\n" + ) + + assert discr.norm(rho_resid, np.inf) < tolerance + assert discr.norm(rhoe_resid, np.inf) < tolerance + for i in range(dim): + assert discr.norm(mom_resid[i], np.inf) < tolerance + for i in range(nspecies): + assert discr.norm(rhoy_resid[i], np.inf) < tolerance + + err_max = discr.norm(rho_resid, np.inf) + eoc_rec0.add_data_point(1.0 / nel_1d, err_max) + + # set a non-zero, but uniform velocity component + for i in range(len(mom_input)): + mom_input[i] = discr.zeros(actx) + (-1.0) ** i + + cv = make_conserved( + dim, mass=mass_input, energy=energy_input, momentum=mom_input, + species_mass=species_mass_input) + + boundaries = {BTAG_ALL: DummyBoundary()} + ns_rhs = ns_operator(discr, eos=eos, boundaries=boundaries, cv=cv, t=0.0) + rhs_resid = ns_rhs - expected_rhs + + rho_resid = rhs_resid.mass + rhoe_resid = rhs_resid.energy + mom_resid = rhs_resid.momentum + rhoy_resid = rhs_resid.species_mass + + assert discr.norm(rho_resid, np.inf) < tolerance + assert discr.norm(rhoe_resid, np.inf) < tolerance + + for i in range(dim): + assert discr.norm(mom_resid[i], np.inf) < tolerance + for i in range(nspecies): + assert discr.norm(rhoy_resid[i], np.inf) < tolerance + + err_max = discr.norm(rho_resid, np.inf) + eoc_rec1.add_data_point(1.0 / nel_1d, err_max) + + logger.info( + f"V == 0 Errors:\n{eoc_rec0}" + f"V != 0 Errors:\n{eoc_rec1}" + ) + + assert ( + eoc_rec0.order_estimate() >= order - 0.5 + or eoc_rec0.max_error() < 1e-9 + ) + assert ( + eoc_rec1.order_estimate() >= order - 0.5 + or eoc_rec1.max_error() < 1e-9 + ) + + +# Box grid generator widget lifted from @majosm and slightly bent +def _get_box_mesh(dim, a, b, n, t=None): + dim_names = ["x", "y", "z"] + bttf = {} + for i in range(dim): + bttf["-"+str(i+1)] = ["-"+dim_names[i]] + bttf["+"+str(i+1)] = ["+"+dim_names[i]] + from meshmode.mesh.generation import generate_regular_rect_mesh as gen + return gen(a=a, b=b, n=n, boundary_tag_to_face=bttf, mesh_type=t) + + +@pytest.mark.parametrize("order", [2, 3]) +def test_poiseuille_rhs(actx_factory, order): + """Test the Navier-Stokes operator using a Poiseuille state. + + This state should yield rhs = 0 to FP. The test is performed for 1, 2, + and 3 dimensions, with orders 1, 2, and 3, with and without passive species. + """ + actx = actx_factory() + dim = 2 + tolerance = 1e-9 + + from pytools.convergence import EOCRecorder + eoc_rec = EOCRecorder() + + base_pressure = 100000.0 + pressure_ratio = 1.001 + mu = 1.0 + left_boundary_location = 0 + right_boundary_location = 0.1 + ybottom = 0. + ytop = .02 + nspecies = 0 + mu = 1.0 + kappa = 0.0 + spec_diffusivity = 0 * np.ones(nspecies) + transport_model = SimpleTransport(viscosity=mu, thermal_conductivity=kappa, + species_diffusivity=spec_diffusivity) + + eos = IdealSingleGas(transport_model=transport_model) + + def poiseuille_2d(x_vec, eos, cv=None, **kwargs): + y = x_vec[1] + x = x_vec[0] + x0 = left_boundary_location + xmax = right_boundary_location + xlen = xmax - x0 + p_low = base_pressure + p_hi = pressure_ratio*base_pressure + dp = p_hi - p_low + dpdx = dp/xlen + h = ytop - ybottom + u_x = dpdx*y*(h - y)/(2*mu) + p_x = p_hi - dpdx*x + rho = 1.0 + mass = 0*x + rho + u_y = 0*x + velocity = make_obj_array([u_x, u_y]) + ke = .5*np.dot(velocity, velocity)*mass + gamma = eos.gamma() + if cv is not None: + mass = cv.mass + vel = cv.velocity + ke = .5*np.dot(vel, vel)*mass + + rho_e = p_x/(gamma-1) + ke + return make_conserved(2, mass=mass, energy=rho_e, + momentum=mass*velocity) + + initializer = poiseuille_2d + + # for nel_1d in [4, 8, 12]: + for nfac in [1, 2, 4, 8]: + + npts_axis = nfac*(12, 20) + box_ll = (left_boundary_location, ybottom) + box_ur = (right_boundary_location, ytop) + mesh = _get_box_mesh(2, a=box_ll, b=box_ur, n=npts_axis) + + logger.info( + f"Number of {dim}d elements: {mesh.nelements}" + ) + + discr = EagerDGDiscretization(actx, mesh, order=order) + nodes = thaw(actx, discr.nodes()) + + cv_input = initializer(x_vec=nodes, eos=eos) + num_eqns = dim + 2 + expected_rhs = make_conserved( + dim, q=make_obj_array([discr.zeros(actx) + for i in range(num_eqns)]) + ) + boundaries = { + DTAG_BOUNDARY("-1"): PrescribedViscousBoundary(q_func=initializer), + DTAG_BOUNDARY("+1"): PrescribedViscousBoundary(q_func=initializer), + DTAG_BOUNDARY("-2"): IsothermalNoSlipBoundary(), + DTAG_BOUNDARY("+2"): IsothermalNoSlipBoundary()} + + ns_rhs = ns_operator(discr, eos=eos, boundaries=boundaries, cv=cv_input, + t=0.0) + + rhs_resid = ns_rhs - expected_rhs + rho_resid = rhs_resid.mass + # rhoe_resid = rhs_resid.energy + mom_resid = rhs_resid.momentum + + rho_rhs = ns_rhs.mass + # rhoe_rhs = ns_rhs.energy + rhov_rhs = ns_rhs.momentum + # rhoy_rhs = ns_rhs.species_mass + + print( + f"rho_rhs = {rho_rhs}\n" + # f"rhoe_rhs = {rhoe_rhs}\n" + f"rhov_rhs = {rhov_rhs}\n" + # f"rhoy_rhs = {rhoy_rhs}\n" + ) + + tol_fudge = 2e-4 + assert discr.norm(rho_resid, np.inf) < tolerance + # assert discr.norm(rhoe_resid, np.inf) < tolerance + mom_err = [discr.norm(mom_resid[i], np.inf) for i in range(dim)] + err_max = max(mom_err) + for i in range(dim): + assert mom_err[i] < tol_fudge + + # err_max = discr.norm(rho_resid, np.inf) + eoc_rec.add_data_point(1.0 / nfac, err_max) + + logger.info( + f"V != 0 Errors:\n{eoc_rec}" + ) + + assert ( + eoc_rec.order_estimate() >= order - 0.5 + or eoc_rec.max_error() < tol_fudge + ) diff --git a/test/test_operators.py b/test/test_operators.py new file mode 100644 index 000000000..5453080ac --- /dev/null +++ b/test/test_operators.py @@ -0,0 +1,287 @@ +"""Test the operator module for sanity.""" + +__copyright__ = """ +Copyright (C) 2021 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import numpy as np # noqa +import pytest # noqa +import logging +from meshmode.array_context import ( # noqa + pytest_generate_tests_for_pyopencl_array_context + as pytest_generate_tests +) +from pytools.obj_array import make_obj_array +from meshmode.dof_array import thaw +from meshmode.mesh import BTAG_ALL +from mirgecom.flux import gradient_flux_central +from mirgecom.fluid import ( + ConservedVars, + make_conserved +) +from grudge.eager import ( + EagerDGDiscretization, + interior_trace_pair +) +from functools import partial + +logger = logging.getLogger(__name__) + + +def _elbnd_flux(discr, compute_interior_flux, compute_boundary_flux, + int_tpair, boundaries): + return (compute_interior_flux(int_tpair) + + sum(compute_boundary_flux(btag) for btag in boundaries)) + + +# Box grid generator widget lifted from @majosm and slightly bent +def _get_box_mesh(dim, a, b, n, t=None): + dim_names = ["x", "y", "z"] + bttf = {} + for i in range(dim): + bttf["-"+str(i+1)] = ["-"+dim_names[i]] + bttf["+"+str(i+1)] = ["+"+dim_names[i]] + from meshmode.mesh.generation import generate_regular_rect_mesh as gen + return gen(a=a, b=b, npoints_per_axis=n, boundary_tag_to_face=bttf, mesh_type=t) + + +def _coord_test_func(actx=None, x_vec=None, order=1, fac=1.0, grad=False): + """Make a coordinate-based test function or its gradient. + + Test Function + ------------- + 1d: fac*x^order + 2d: fac*x^order * y^order + 3d: fac*x^order * y^order * z^order + + Test Function Gradient + ---------------------- + 1d: fac*order*x^(order-1) + 2d: fac*order* + 3d: fac*order* + """ + if x_vec is None: + return 0 + + dim = len(x_vec) + if grad: + ret_ary = fac*order*make_obj_array([0*x_vec[0]+1.0 for _ in range(dim)]) + for i in range(dim): + for j in range(dim): + termpow = (order - 1) if order and j == i else order + ret_ary[i] = ret_ary[i] * (x_vec[j]**termpow) + else: + ret_ary = fac*(0*x_vec[0]+1.0) + for i in range(dim): + ret_ary = ret_ary * (x_vec[i]**order) + + return ret_ary + + +def _trig_test_func(actx=None, x_vec=None, grad=False): + """Make trig test function or its gradient. + + Test Function + ------------- + 1d: cos(2pi x) + 2d: sin(2pi x)cos(2pi y) + 3d: sin(2pi x)sin(2pi y)cos(2pi x) + + Grad Test Function + ------------------ + 1d: 2pi * -sin(2pi x) + 2d: 2pi * + 3d: 2pi * + """ + if x_vec is None: + return 0 + dim = len(x_vec) + if grad: + ret_ary = make_obj_array([0*x_vec[0] + 1.0 for _ in range(dim)]) + for i in range(dim): # component & derivative for ith dir + for j in range(dim): # form term for jth dir in ith component + if j == i: # then this is a derivative term + if j == (dim-1): # deriv of cos term + ret_ary[i] = ret_ary[i] * -actx.np.sin(2*np.pi*x_vec[j]) + else: # deriv of sin term + ret_ary[i] = ret_ary[i] * actx.np.cos(2*np.pi*x_vec[j]) + ret_ary[i] = 2*np.pi*ret_ary[i] + else: # non-derivative term + if j == (dim-1): # cos term + ret_ary[i] = ret_ary[i] * actx.np.cos(2*np.pi*x_vec[j]) + else: # sin term + ret_ary[i] = ret_ary[i] * actx.np.sin(2*np.pi*x_vec[j]) + else: + # return _make_trig_term(actx, r=x_vec, term=dim-1) + ret_ary = actx.np.cos(2*np.pi*x_vec[dim-1]) + for i in range(dim-1): + ret_ary = ret_ary * actx.np.sin(2*np.pi*x_vec[i]) + + return ret_ary + + +def _cv_test_func(actx, x_vec, grad=False): + """Make a CV array container for testing. + + There is no need for this CV to be physical, we are just testing whether the + operator machinery still gets the right answers when operating on a CV array + container. + + Testing CV + ---------- + mass = constant + energy = _trig_test_func + momentum = <_coord_test_func(order=2, fac=dim_index+1)> + + Testing CV Gradient + ------------------- + mass = <0> + energy = <_trig_test_func(grad=True)> + momentum = <_coord_test_func(grad=True)> + """ + zeros = 0*x_vec[0] + dim = len(x_vec) + momentum = make_obj_array([zeros+1.0 for _ in range(dim)]) + if grad: + dm = make_obj_array([zeros+0.0 for _ in range(dim)]) + de = _trig_test_func(actx, x_vec, grad=True) + dp = make_obj_array([np.empty(0) for _ in range(dim)]) + dy = ( + momentum * make_obj_array([np.empty(0) for _ in range(0)]).reshape(-1, 1) + ) + for i in range(dim): + dp[i] = _coord_test_func(actx, x_vec, order=2, fac=(i+1), grad=True) + return make_conserved(dim=dim, mass=dm, energy=de, momentum=np.stack(dp), + species_mass=dy) + else: + mass = zeros + 1.0 + energy = _trig_test_func(actx, x_vec) + for i in range(dim): + momentum[i] = _coord_test_func(actx, x_vec, order=2, fac=(i+1)) + return make_conserved(dim=dim, mass=mass, energy=energy, momentum=momentum) + + +def central_flux_interior(actx, discr, int_tpair): + """Compute a central flux for interior faces.""" + normal = thaw(actx, discr.normal(int_tpair.dd)) + flux_weak = gradient_flux_central(int_tpair, normal) + return discr.project(int_tpair.dd, "all_faces", flux_weak) + + +def central_flux_boundary(actx, discr, soln_func, btag): + """Compute a central flux for boundary faces.""" + boundary_discr = discr.discr_from_dd(btag) + bnd_nodes = thaw(actx, boundary_discr.nodes()) + soln_bnd = soln_func(actx=actx, x_vec=bnd_nodes) + bnd_nhat = thaw(actx, discr.normal(btag)) + from grudge.trace_pair import TracePair + bnd_tpair = TracePair(btag, interior=soln_bnd, exterior=soln_bnd) + flux_weak = gradient_flux_central(bnd_tpair, bnd_nhat) + return discr.project(bnd_tpair.dd, "all_faces", flux_weak) + + +@pytest.mark.parametrize("dim", [1, 2, 3]) +@pytest.mark.parametrize("order", [1, 2, 3]) +@pytest.mark.parametrize("test_func", [partial(_coord_test_func, order=0), + partial(_coord_test_func, order=1), + partial(_coord_test_func, order=1, fac=2), + partial(_coord_test_func, order=2), + _trig_test_func, + _cv_test_func]) +def test_grad_operator(actx_factory, dim, order, test_func): + """Test the gradient operator for sanity. + + Check whether we get the right answers for gradients of analytic functions with + some simple input fields and states: + - constant + - multilinear funcs + - quadratic funcs + - trig funcs + - :class:`~mirgecom.fluid.ConservedVars` composed of funcs from above + """ + actx = actx_factory() + + tol = 1e-10 if dim < 3 else 1e-9 + from pytools.convergence import EOCRecorder + eoc = EOCRecorder() + + for nfac in [1, 2, 4]: + + npts_axis = (nfac*4,)*dim + box_ll = (0,)*dim + box_ur = (1,)*dim + mesh = _get_box_mesh(dim, a=box_ll, b=box_ur, n=npts_axis) + + logger.info( + f"Number of {dim}d elements: {mesh.nelements}" + ) + + discr = EagerDGDiscretization(actx, mesh, order=order) + # compute max element size + from grudge.dt_utils import h_max_from_volume + h_max = h_max_from_volume(discr) + + nodes = thaw(actx, discr.nodes()) + int_flux = partial(central_flux_interior, actx, discr) + bnd_flux = partial(central_flux_boundary, actx, discr, test_func) + + test_data = test_func(actx=actx, x_vec=nodes) + exact_grad = test_func(actx=actx, x_vec=nodes, grad=True) + + if isinstance(test_data, ConservedVars): + err_scale = discr.norm(exact_grad.join(), np.inf) + else: + err_scale = discr.norm(exact_grad, np.inf) + if err_scale <= 1e-16: + err_scale = 1 + + print(f"{test_data=}") + print(f"{exact_grad=}") + + test_data_int_tpair = interior_trace_pair(discr, test_data) + boundaries = [BTAG_ALL] + test_data_flux_bnd = _elbnd_flux(discr, int_flux, bnd_flux, + test_data_int_tpair, boundaries) + + from mirgecom.operators import grad_operator + if isinstance(test_data, ConservedVars): + test_grad = make_conserved( + dim=dim, q=grad_operator(discr, test_data.join(), + test_data_flux_bnd.join()) + ) + grad_err = discr.norm((test_grad - exact_grad).join(), np.inf)/err_scale + else: + test_grad = grad_operator(discr, test_data, test_data_flux_bnd) + grad_err = discr.norm(test_grad - exact_grad, np.inf)/err_scale + + print(f"{test_grad=}") + eoc.add_data_point(h_max, grad_err) + + assert ( + eoc.order_estimate() >= order - 0.5 + or eoc.max_error() < tol + ) diff --git a/test/test_viscous.py b/test/test_viscous.py index 6cb92ad6f..369231075 100644 --- a/test/test_viscous.py +++ b/test/test_viscous.py @@ -33,8 +33,12 @@ from pytools.obj_array import make_obj_array from meshmode.dof_array import thaw +from meshmode.mesh import BTAG_ALL import grudge.op as op -from grudge.eager import EagerDGDiscretization +from grudge.eager import ( + EagerDGDiscretization, + interior_trace_pair +) from meshmode.array_context import ( # noqa pytest_generate_tests_for_pyopencl_array_context as pytest_generate_tests) @@ -49,35 +53,17 @@ logger = logging.getLogger(__name__) -def dont_test_actx_power(actx_factory): - """Test power of DOFArrays and the likes.""" - actx = actx_factory() - dim = 3 - nel_1d = 5 - from meshmode.mesh.generation import generate_regular_rect_mesh - mesh = generate_regular_rect_mesh( - a=(1.0,) * dim, b=(2.0,) * dim, n=(nel_1d,) * dim - ) - order = 1 - discr = EagerDGDiscretization(actx, mesh, order=order) - nodes = thaw(actx, discr.nodes()) - - actx.np.power(nodes, .5) - actx.np.power(nodes[0], .5) - actx.np.power(nodes[0][0], .5) - - # TODO: Bring back transport_model 0 when *actx.np.power* is fixed @pytest.mark.parametrize("transport_model", [1]) def test_viscous_stress_tensor(actx_factory, transport_model): """Test tau data structure and values against exact.""" actx = actx_factory() dim = 3 - nel_1d = 5 + nel_1d = 4 from meshmode.mesh.generation import generate_regular_rect_mesh mesh = generate_regular_rect_mesh( - a=(1.0,) * dim, b=(2.0,) * dim, n=(nel_1d,) * dim + a=(1.0,) * dim, b=(2.0,) * dim, nelements_per_axis=(nel_1d,) * dim ) order = 1 @@ -123,16 +109,180 @@ def test_viscous_stress_tensor(actx_factory, transport_model): assert discr.norm(tau - exp_tau, np.inf) < 1e-12 +# Box grid generator widget lifted from @majosm and slightly bent +def _get_box_mesh(dim, a, b, n, t=None): + dim_names = ["x", "y", "z"] + bttf = {} + for i in range(dim): + bttf["-"+str(i+1)] = ["-"+dim_names[i]] + bttf["+"+str(i+1)] = ["+"+dim_names[i]] + from meshmode.mesh.generation import generate_regular_rect_mesh as gen + return gen(a=a, b=b, npoints_per_axis=n, boundary_tag_to_face=bttf, mesh_type=t) + + +@pytest.mark.parametrize("order", [2, 3, 4]) +@pytest.mark.parametrize("kappa", [0.0, 1.0, 2.3]) +def test_poiseuille_fluxes(actx_factory, order, kappa): + """Test the viscous fluxes using a Poiseuille input state.""" + actx = actx_factory() + dim = 2 + + from pytools.convergence import EOCRecorder + e_eoc_rec = EOCRecorder() + p_eoc_rec = EOCRecorder() + + base_pressure = 100000.0 + pressure_ratio = 1.001 + mu = 42 # arbitrary + left_boundary_location = 0 + right_boundary_location = 0.1 + ybottom = 0. + ytop = .02 + nspecies = 0 + spec_diffusivity = 0 * np.ones(nspecies) + transport_model = SimpleTransport(viscosity=mu, thermal_conductivity=kappa, + species_diffusivity=spec_diffusivity) + + xlen = right_boundary_location - left_boundary_location + p_low = base_pressure + p_hi = pressure_ratio*base_pressure + dpdx = (p_hi - p_low) / xlen + rho = 1.0 + + eos = IdealSingleGas(transport_model=transport_model) + + from mirgecom.initializers import PlanarPoiseuille + initializer = PlanarPoiseuille(density=rho, mu=mu) + + def _elbnd_flux(discr, compute_interior_flux, compute_boundary_flux, + int_tpair, boundaries): + return (compute_interior_flux(int_tpair) + + sum(compute_boundary_flux(btag) for btag in boundaries)) + + from mirgecom.flux import gradient_flux_central + + def cv_flux_interior(int_tpair): + normal = thaw(actx, discr.normal(int_tpair.dd)) + flux_weak = gradient_flux_central(int_tpair, normal) + return discr.project(int_tpair.dd, "all_faces", flux_weak) + + def cv_flux_boundary(btag): + boundary_discr = discr.discr_from_dd(btag) + bnd_nodes = thaw(actx, boundary_discr.nodes()) + cv_bnd = initializer(x_vec=bnd_nodes, eos=eos) + bnd_nhat = thaw(actx, discr.normal(btag)) + from grudge.trace_pair import TracePair + bnd_tpair = TracePair(btag, interior=cv_bnd, exterior=cv_bnd) + flux_weak = gradient_flux_central(bnd_tpair, bnd_nhat) + return discr.project(bnd_tpair.dd, "all_faces", flux_weak) + + for nfac in [1, 2, 4]: + + npts_axis = nfac*(11, 21) + box_ll = (left_boundary_location, ybottom) + box_ur = (right_boundary_location, ytop) + mesh = _get_box_mesh(2, a=box_ll, b=box_ur, n=npts_axis) + + logger.info( + f"Number of {dim}d elements: {mesh.nelements}" + ) + + discr = EagerDGDiscretization(actx, mesh, order=order) + nodes = thaw(actx, discr.nodes()) + + # compute max element size + from grudge.dt_utils import h_max_from_volume + h_max = h_max_from_volume(discr) + + # form exact cv + cv = initializer(x_vec=nodes, eos=eos) + cv_int_tpair = interior_trace_pair(discr, cv) + boundaries = [BTAG_ALL] + cv_flux_bnd = _elbnd_flux(discr, cv_flux_interior, cv_flux_boundary, + cv_int_tpair, boundaries) + from mirgecom.operators import grad_operator + grad_cv = make_conserved(dim, q=grad_operator(discr, cv.join(), + cv_flux_bnd.join())) + + xp_grad_cv = initializer.exact_grad(x_vec=nodes, eos=eos, cv_exact=cv) + xp_grad_v = 1/cv.mass * xp_grad_cv.momentum + xp_tau = mu * (xp_grad_v + xp_grad_v.transpose()) + + # sanity check the gradient: + relerr_scale_e = 1.0 / discr.norm(xp_grad_cv.energy, np.inf) + relerr_scale_p = 1.0 / discr.norm(xp_grad_cv.momentum, np.inf) + graderr_e = discr.norm((grad_cv.energy - xp_grad_cv.energy), np.inf) + graderr_p = discr.norm((grad_cv.momentum - xp_grad_cv.momentum), np.inf) + graderr_e *= relerr_scale_e + graderr_p *= relerr_scale_p + assert graderr_e < 5e-7 + assert graderr_p < 5e-11 + + zeros = discr.zeros(actx) + ones = zeros + 1 + pressure = eos.pressure(cv) + # grad of p should be dp/dx + xp_grad_p = -make_obj_array([dpdx*ones, zeros]) + grad_p = op.local_grad(discr, pressure) + + temperature = eos.temperature(cv) + tscal = dpdx/(rho*eos.gas_const()) + xp_grad_t = xp_grad_p/(cv.mass*eos.gas_const()) + grad_t = op.local_grad(discr, temperature) + + # sanity check + assert discr.norm(grad_p - xp_grad_p, np.inf)/dpdx < 5e-9 + assert discr.norm(grad_t - xp_grad_t, np.inf)/tscal < 5e-9 + + # verify heat flux + from mirgecom.viscous import conductive_heat_flux + heat_flux = conductive_heat_flux(discr, eos, cv, grad_t) + xp_heat_flux = -kappa*xp_grad_t + assert discr.norm(heat_flux - xp_heat_flux, np.inf) < 2e-8 + + # verify diffusive mass flux is zilch (no scalar components) + from mirgecom.viscous import diffusive_flux + j = diffusive_flux(discr, eos, cv, grad_cv) + assert len(j) == 0 + + xp_e_flux = np.dot(xp_tau, cv.velocity) - xp_heat_flux + xp_mom_flux = xp_tau + from mirgecom.viscous import viscous_flux + vflux = viscous_flux(discr, eos, cv, grad_cv, grad_t) + + efluxerr = ( + discr.norm(vflux.energy - xp_e_flux, np.inf) + / discr.norm(xp_e_flux, np.inf) + ) + momfluxerr = ( + discr.norm(vflux.momentum - xp_mom_flux, np.inf) + / discr.norm(xp_mom_flux, np.inf) + ) + + assert discr.norm(vflux.mass, np.inf) == 0 + e_eoc_rec.add_data_point(h_max, efluxerr) + p_eoc_rec.add_data_point(h_max, momfluxerr) + + assert ( + e_eoc_rec.order_estimate() >= order - 0.5 + or e_eoc_rec.max_error() < 3e-9 + ) + assert ( + p_eoc_rec.order_estimate() >= order - 0.5 + or p_eoc_rec.max_error() < 2e-12 + ) + + def test_species_diffusive_flux(actx_factory): """Test species diffusive flux and values against exact.""" actx = actx_factory() dim = 3 - nel_1d = 5 + nel_1d = 4 from meshmode.mesh.generation import generate_regular_rect_mesh mesh = generate_regular_rect_mesh( - a=(1.0,) * dim, b=(2.0,) * dim, n=(nel_1d,) * dim + a=(1.0,) * dim, b=(2.0,) * dim, nelements_per_axis=(nel_1d,) * dim ) order = 1 @@ -198,12 +348,12 @@ def test_diffusive_heat_flux(actx_factory): """Test diffusive heat flux and values against exact.""" actx = actx_factory() dim = 3 - nel_1d = 5 + nel_1d = 4 from meshmode.mesh.generation import generate_regular_rect_mesh mesh = generate_regular_rect_mesh( - a=(1.0,) * dim, b=(2.0,) * dim, n=(nel_1d,) * dim + a=(1.0,) * dim, b=(2.0,) * dim, nelements_per_axis=(nel_1d,) * dim ) order = 1 @@ -312,12 +462,12 @@ def test_local_max_species_diffusivity(actx_factory, dim): def test_viscous_timestep(actx_factory, dim, mu, vel): """Test timestep size.""" actx = actx_factory() - nel_1d = 5 + nel_1d = 4 from meshmode.mesh.generation import generate_regular_rect_mesh mesh = generate_regular_rect_mesh( - a=(1.0,) * dim, b=(2.0,) * dim, n=(nel_1d,) * dim + a=(1.0,) * dim, b=(2.0,) * dim, nelements_per_axis=(nel_1d,) * dim ) order = 1 From d10d05990d6f587b73c5895dec0098f453d58547 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Sat, 21 Aug 2021 14:38:00 -0500 Subject: [PATCH 303/385] Remove gmsh from requirements --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 326fe7806..afefc2614 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,6 @@ pymetis importlib-resources psutil pyyaml -gmsh # The following packages will be git cloned by emirge: --editable git+https://github.com/inducer/pymbolic.git#egg=pymbolic From 913eea9f8ed1a0597f432ccd82aa0ec41441ea2e Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sun, 22 Aug 2021 08:12:20 -0500 Subject: [PATCH 304/385] Fix up power law transport to be functional. --- mirgecom/transport.py | 8 +------- test/test_viscous.py | 3 +-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/mirgecom/transport.py b/mirgecom/transport.py index 520f2b592..071c1504d 100644 --- a/mirgecom/transport.py +++ b/mirgecom/transport.py @@ -170,9 +170,6 @@ class PowerLawTransport(TransportModel): def __init__(self, alpha=0.6, beta=4.093e-7, sigma=2.5, n=.666, species_diffusivity=None): """Initialize power law coefficients and parameters.""" - raise NotImplementedError("This class is not yet supported, awaits " - "implementation of array_context.np.power.") - if species_diffusivity is None: species_diffusivity = np.empty((0,), dtype=object) self._alpha = alpha @@ -194,10 +191,7 @@ def viscosity(self, eos: GasEOS, cv: ConservedVars): $\mu = \beta{T}^n$ """ - actx = cv.array_context - gas_t = eos.temperature(cv) - # TODO: actx.np.power is unimplemented - return self._beta * actx.np.power(gas_t, self._n) + return self._beta * eos.temperature(cv)**self._n def volume_viscosity(self, eos: GasEOS, cv: ConservedVars): r"""Get the 2nd viscosity coefficent, $\lambda$. diff --git a/test/test_viscous.py b/test/test_viscous.py index 369231075..d68e6a368 100644 --- a/test/test_viscous.py +++ b/test/test_viscous.py @@ -53,8 +53,7 @@ logger = logging.getLogger(__name__) -# TODO: Bring back transport_model 0 when *actx.np.power* is fixed -@pytest.mark.parametrize("transport_model", [1]) +@pytest.mark.parametrize("transport_model", [0, 1]) def test_viscous_stress_tensor(actx_factory, transport_model): """Test tau data structure and values against exact.""" actx = actx_factory() From 954348d2ab92638309eba136382a2aae321741e8 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sun, 22 Aug 2021 20:38:35 -0500 Subject: [PATCH 305/385] Update doublemach example with developments from upstream --- examples/doublemach-mpi.py | 162 ++++++++++++++++++++++--------------- 1 file changed, 99 insertions(+), 63 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 206973fe8..4cbf50efd 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -30,7 +30,12 @@ import pyopencl.tools as cl_tools from functools import partial -from meshmode.array_context import PyOpenCLArrayContext +from meshmode.array_context import ( + PyOpenCLArrayContext, + PytatoPyOpenCLArrayContext +) +from mirgecom.profiling import PyOpenCLProfilingArrayContext + from meshmode.dof_array import thaw from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa from grudge.dof_desc import DTAG_BOUNDARY @@ -38,7 +43,6 @@ from grudge.shortcuts import make_visualizer -from mirgecom.fluid import make_conserved from mirgecom.navierstokes import ns_operator from mirgecom.artificial_viscosity import ( av_operator, @@ -46,7 +50,7 @@ ) from mirgecom.io import make_init_message from mirgecom.mpi import mpi_entry_point - +from mirgecom.fluid import make_conserved from mirgecom.integrators import rk4_step from mirgecom.steppers import advance_state from mirgecom.boundary import ( @@ -56,10 +60,10 @@ from mirgecom.initializers import DoubleMachReflection from mirgecom.eos import IdealSingleGas from mirgecom.transport import SimpleTransport +from mirgecom.simutil import get_sim_timestep from logpyle import set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging -from mirgecom.profiling import PyOpenCLProfilingArrayContext from mirgecom.logging_quantities import ( initialize_logmgr, logmgr_add_many_discretization_quantities, @@ -125,9 +129,9 @@ def get_doublemach_mesh(): @mpi_entry_point -def main(ctx_factory=cl.create_some_context, use_leap=False, - use_profiling=False, rst_step=None, rst_name=None, - casename="doubleMach", use_logmgr=True): +def main(ctx_factory=cl.create_some_context, use_logmgr=True, + use_leap=False, use_profiling=False, casename=None, + rst_filename=None, actx_class=PyOpenCLArrayContext): """Drive the example.""" cl_ctx = ctx_factory() @@ -143,61 +147,39 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) if use_profiling: - queue = cl.CommandQueue(cl_ctx, - properties=cl.command_queue_properties.PROFILING_ENABLE) - actx = PyOpenCLProfilingArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue)), - logmgr=logmgr) + queue = cl.CommandQueue( + cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) else: queue = cl.CommandQueue(cl_ctx) - actx = PyOpenCLArrayContext(queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - dim = 2 - order = 3 - # Too many steps for CI - # t_final = 1.0e-2 + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + # Timestepping control + current_step = 0 + timestepper = rk4_step t_final = 1.0e-3 current_cfl = 0.1 current_dt = 1.0e-4 current_t = 0 - # {{{ Initialize simple transport model - kappa = 1e-5 - sigma = 1e-5 - transport_model = SimpleTransport(viscosity=sigma, thermal_conductivity=kappa) - # }}} - eos = IdealSingleGas(transport_model=transport_model) - initializer = DoubleMachReflection() - - boundaries = { - DTAG_BOUNDARY("ic1"): PrescribedBoundary(initializer), - DTAG_BOUNDARY("ic2"): PrescribedBoundary(initializer), - DTAG_BOUNDARY("ic3"): PrescribedBoundary(initializer), - DTAG_BOUNDARY("wall"): AdiabaticNoslipMovingBoundary(), - DTAG_BOUNDARY("out"): AdiabaticNoslipMovingBoundary(), - } constant_cfl = False + + # Some i/o frequencies nstatus = 10 nviz = 100 - current_step = 0 - timestepper = rk4_step nrestart = 100 nhealth = 1 - s0 = -6.0 - kappa = 1.0 - alpha = 2.0e-2 - from mpi4py import MPI - rst_path = "restart_data/" rst_pattern = ( rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" ) - if rst_step: # read the grid from restart data - rst_fname = rst_pattern.format(cname=rst_name, step=rst_step, rank=rank) + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, rst_fname) + restart_data = read_restart_data(actx, rst_filename) local_mesh = restart_data["local_mesh"] local_nelements = local_mesh.nelements global_nelements = restart_data["global_nelements"] @@ -208,10 +190,12 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, local_mesh, global_nelements = generate_and_distribute_mesh(comm, gen_grid) local_nelements = local_mesh.nelements + order = 3 discr = EagerDGDiscretization(actx, local_mesh, order=order, mpi_communicator=comm) nodes = thaw(actx, discr.nodes()) + dim = 2 if logmgr: logmgr_add_device_name(logmgr, queue) logmgr_add_device_memory_usage(logmgr, queue) @@ -229,9 +213,30 @@ def main(ctx_factory=cl.create_some_context, use_leap=False, ("t_log.max", "log walltime: {value:6g} s") ]) - if rst_step: + # Solution setup and initialization + s0 = -6.0 + kappa = 1.0 + alpha = 2.0e-2 + # {{{ Initialize simple transport model + kappa_t = 1e-5 + sigma_v = 1e-5 + transport_model = SimpleTransport(viscosity=sigma_v, + thermal_conductivity=kappa_t) + # }}} + eos = IdealSingleGas(transport_model=transport_model) + initializer = DoubleMachReflection() + + boundaries = { + DTAG_BOUNDARY("ic1"): PrescribedBoundary(initializer), + DTAG_BOUNDARY("ic2"): PrescribedBoundary(initializer), + DTAG_BOUNDARY("ic3"): PrescribedBoundary(initializer), + DTAG_BOUNDARY("wall"): AdiabaticNoslipMovingBoundary(), + DTAG_BOUNDARY("out"): AdiabaticNoslipMovingBoundary(), + } + + if rst_filename: current_t = restart_data["t"] - current_step = rst_step + current_step = restart_data["step"] current_state = restart_data["state"] if logmgr: from mirgecom.logging_quantities import logmgr_set_time @@ -278,17 +283,18 @@ def my_write_viz(step, t, state, dv=None, tagged_cells=None): def my_write_restart(step, t, state): rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) - rst_data = { - "local_mesh": local_mesh, - "state": state, - "t": t, - "step": step, - "order": order, - "global_nelements": global_nelements, - "num_parts": nparts - } - from mirgecom.restart import write_restart_file - write_restart_file(actx, rst_data, rst_fname, comm) + if rst_fname != rst_filename: + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": nparts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) def my_health_check(state, dv): # Note: This health check is tuned s.t. it is a test that @@ -347,10 +353,6 @@ def my_pre_step(step, t, dt, state): logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") - if step == rst_step: # don't do viz or restart @ restart - do_viz = False - do_restart = False - if do_restart: my_write_restart(step=step, t=t, state=state) @@ -369,8 +371,10 @@ def my_pre_step(step, t, dt, state): my_write_restart(step=step, t=t, state=state) raise - t_remaining = max(0, t_final - t) - return state, min(dt, t_remaining) + dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + + return state, dt def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? @@ -390,6 +394,9 @@ def my_rhs(t, state): s0=s0, kappa=kappa) ) + current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + current_step, current_t, current_state = \ advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, @@ -413,7 +420,36 @@ def my_rhs(t, state): if __name__ == "__main__": + import argparse + casename = "doublemach" + parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") + parser.add_argument("--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("--log", action="store_true", default=True, + help="turn on logging") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--restart_file", help="root name of restart file") + parser.add_argument("--casename", help="casename to use for i/o") + args = parser.parse_args() + if args.profiling: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext + logging.basicConfig(format="%(message)s", level=logging.INFO) - main() + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling, + casename=casename, rst_filename=rst_filename, actx_class=actx_class) # vim: foldmethod=marker From c65c653dfeee4791215abb42d46b92587138c785 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 23 Aug 2021 07:21:46 -0500 Subject: [PATCH 306/385] Fix bug in av boundary routine --- mirgecom/boundary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index b2a2fb6ca..18364b3d4 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -380,7 +380,7 @@ def adiabatic_slip_pair(self, discr, cv, btag, **kwargs): def exterior_grad_q(self, nodes, nhat, grad_cv, **kwargs): """Get the exterior grad(Q) on the boundary.""" # Grab some boundary-relevant data - num_equations, dim = grad_cv.mass.shape + dim, = grad_cv.mass.shape # Subtract 2*wall-normal component of q # to enforce q=0 on the wall From e999d03e660820e55bf22d68876ebcdfef2da275 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 23 Aug 2021 07:32:17 -0500 Subject: [PATCH 307/385] Remove whitespace to match upstream --- mirgecom/eos.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mirgecom/eos.py b/mirgecom/eos.py index ec604da92..8c019d7b1 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -695,7 +695,6 @@ def get_pressure(): temperature = self.temperature(cv) y = cv.species_mass_fractions return self._pyrometheus_mech.get_pressure(cv.mass, temperature, y) - return get_pressure() def sound_speed(self, cv: ConservedVars): @@ -724,7 +723,6 @@ def sound_speed(self, cv: ConservedVars): def get_sos(): actx = cv.array_context return actx.np.sqrt((self.gamma(cv) * self.pressure(cv)) / cv.mass) - return get_sos() def temperature(self, cv: ConservedVars): @@ -757,7 +755,6 @@ def get_temp(): e = self.internal_energy(cv) / cv.mass return self._pyrometheus_mech.get_temperature(e, self._tguess, y, True) - return get_temp() def total_energy(self, cv, pressure): From 124ac0b4959dca445bb5b57562966c63d8acf9c5 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 26 Aug 2021 08:21:52 -0500 Subject: [PATCH 308/385] Disable some portions of viscous flux temporarily for performance test. --- mirgecom/viscous.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mirgecom/viscous.py b/mirgecom/viscous.py index 1b371e803..46db4fc16 100644 --- a/mirgecom/viscous.py +++ b/mirgecom/viscous.py @@ -257,7 +257,8 @@ def viscous_flux(discr, eos, cv, grad_cv, grad_t): viscous_mass_flux = 0 * cv.momentum j = diffusive_flux(discr, eos, cv, grad_cv) - heat_flux_diffusive = diffusive_heat_flux(discr, eos, cv, j) + # heat_flux_diffusive = diffusive_heat_flux(discr, eos, cv, j) + heat_flux_diffusive = 0 tau = viscous_stress_tensor(discr, eos, cv, grad_cv) viscous_energy_flux = ( @@ -347,11 +348,11 @@ def get_viscous_timestep(discr, eos, cv): transport = eos.transport_model() if transport: mu = transport.viscosity(eos, cv) - d_alpha_max = \ - get_local_max_species_diffusivity( - cv.array_context, discr, - transport.species_diffusivity(eos, cv) - ) + # d_alpha_max = \ + # get_local_max_species_diffusivity( + # cv.array_context, discr, + # transport.species_diffusivity(eos, cv) + # ) return( length_scales / (compute_wavespeed(eos, cv) From 414d24dd38578b6ff5afb66cac729c6e93a930d8 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 30 Aug 2021 10:59:20 -0500 Subject: [PATCH 309/385] Add bozzle case to examples. --- examples/bozzle.py | 863 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 863 insertions(+) create mode 100644 examples/bozzle.py diff --git a/examples/bozzle.py b/examples/bozzle.py new file mode 100644 index 000000000..d903a9d87 --- /dev/null +++ b/examples/bozzle.py @@ -0,0 +1,863 @@ +"""Production-like (nozzle-like) case in a box.""" + +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" +import os +import yaml +import logging +import numpy as np +import pyopencl as cl +import numpy.linalg as la # noqa +import pyopencl.array as cla # noqa +from functools import partial +import math + +from meshmode.dof_array import thaw +from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa +from grudge.dof_desc import DTAG_BOUNDARY +from grudge.eager import EagerDGDiscretization +from grudge.shortcuts import make_visualizer + +from meshmode.array_context import ( + PyOpenCLArrayContext, + SingleGridWorkBalancingPytatoArrayContext as PytatoPyOpenCLArrayContext +) +from mirgecom.profiling import PyOpenCLProfilingArrayContext + +from mirgecom.navierstokes import ns_operator +from mirgecom.fluid import make_conserved +from mirgecom.artificial_viscosity import (av_operator, smoothness_indicator) +from mirgecom.simutil import ( + check_step, + generate_and_distribute_mesh, + write_visfile, + check_naninf_local, + check_range_local, + get_sim_timestep +) +from mirgecom.restart import write_restart_file +from mirgecom.io import make_init_message +from mirgecom.mpi import mpi_entry_point +import pyopencl.tools as cl_tools +# from mirgecom.checkstate import compare_states +from mirgecom.integrators import (rk4_step, lsrk54_step, lsrk144_step, + euler_step) +from mirgecom.steppers import advance_state +from mirgecom.boundary import ( + PrescribedInviscidBoundary, + IsothermalNoSlipBoundary +) +from mirgecom.initializers import (Uniform, PlanarDiscontinuity) +from mirgecom.eos import IdealSingleGas +from mirgecom.transport import SimpleTransport + +from logpyle import IntervalTimer, set_dt +from mirgecom.euler import extract_vars_for_logging, units_for_logging +from mirgecom.logging_quantities import ( + initialize_logmgr, logmgr_add_many_discretization_quantities, + logmgr_add_cl_device_info, logmgr_set_time, LogUserQuantity, + set_sim_state +) + +logger = logging.getLogger(__name__) + + +class MyRuntimeError(RuntimeError): + """Simple exception to kill the simulation.""" + + pass + + +def get_pseudo_y0_mesh(): + """Generate or import a grid using `gmsh`. + + Input required: + data/pseudoY0.brep (for mesh gen) + -or- + data/pseudoY0.msh (read existing mesh) + + This routine will generate a new grid if it does + not find the grid file (data/pseudoY0.msh), but + note that if the grid is generated in millimeters, + then the solution initialization and BCs need to be + adjusted or the grid needs to be scaled up to meters + before being used with the current main driver in this + example. + """ + from meshmode.mesh.io import (read_gmsh, generate_gmsh, + ScriptWithFilesSource) + if os.path.exists("data/pseudoY1nozzle.msh") is False: + mesh = generate_gmsh(ScriptWithFilesSource( + """ + Merge "data/pseudoY1nozzle.brep"; + Mesh.CharacteristicLengthMin = 1; + Mesh.CharacteristicLengthMax = 10; + Mesh.ElementOrder = 2; + Mesh.CharacteristicLengthExtendFromBoundary = 0; + + // Inside and end surfaces of nozzle/scramjet + Field[1] = Distance; + Field[1].NNodesByEdge = 100; + Field[1].FacesList = {5,7,8,9,10}; + Field[2] = Threshold; + Field[2].IField = 1; + Field[2].LcMin = 1; + Field[2].LcMax = 10; + Field[2].DistMin = 0; + Field[2].DistMax = 20; + + // Edges separating surfaces with boundary layer + // refinement from those without + // (Seems to give a smoother transition) + Field[3] = Distance; + Field[3].NNodesByEdge = 100; + Field[3].EdgesList = {5,10,14,16}; + Field[4] = Threshold; + Field[4].IField = 3; + Field[4].LcMin = 1; + Field[4].LcMax = 10; + Field[4].DistMin = 0; + Field[4].DistMax = 20; + + // Min of the two sections above + Field[5] = Min; + Field[5].FieldsList = {2,4}; + + Background Field = 5; + """, ["data/pseudoY1nozzle.brep"]), 3, target_unit="MM") + else: + mesh = read_gmsh("data/pseudoY1nozzle.msh") + + return mesh + + +@mpi_entry_point +def main(ctx_factory=cl.create_some_context, restart_filename=None, + use_profiling=False, use_logmgr=False, user_input_file=None, + actx_class=PyOpenCLArrayContext, casename=None): + """Drive the Y0 nozzle example.""" + cl_ctx = ctx_factory() + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + nparts = comm.Get_size() + + if casename is None: + casename = "mirgecom" + + # logging and profiling + logmgr = initialize_logmgr(use_logmgr, + filename=f"{casename}.sqlite", mode="wo", mpi_comm=comm) + + if use_profiling: + queue = cl.CommandQueue(cl_ctx, + properties=cl.command_queue_properties.PROFILING_ENABLE) + else: + queue = cl.CommandQueue(cl_ctx) + + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + # Most of these can be set by the user input file + + # default i/o junk frequencies + nviz = 100 + nrestart = 100 + nhealth = 100 + nstatus = 1 + log_dependent = 1 + + # default timestepping control + integrator = "rk4" + current_dt = 5.0e-8 + t_final = 5.0e-6 + current_cfl = 1.0 + current_t = 0 + constant_cfl = False + current_step = 0 + + # default health status bounds + health_pres_min = 1.0e-1 + health_pres_max = 2.0e6 + + # discretization and model control + order = 1 + alpha_sc = 0.5 + s0_sc = -5.0 + kappa_sc = 0.5 + weak_scale = 1 + + if user_input_file: + input_data = None + if rank == 0: + with open(user_input_file) as f: + input_data = yaml.load(f, Loader=yaml.FullLoader) + input_data = comm.bcast(input_data, root=0) + try: + weak_scale = float(input_data["wscale"]) + except KeyError: + pass + try: + nviz = int(input_data["nviz"]) + except KeyError: + pass + try: + nrestart = int(input_data["nrestart"]) + except KeyError: + pass + try: + nhealth = int(input_data["nhealth"]) + except KeyError: + pass + try: + nstatus = int(input_data["nstatus"]) + except KeyError: + pass + try: + log_dependent = int(input_data["logDependent"]) + except KeyError: + pass + try: + current_dt = float(input_data["current_dt"]) + except KeyError: + pass + try: + t_final = float(input_data["t_final"]) + except KeyError: + pass + try: + alpha_sc = float(input_data["alpha_sc"]) + except KeyError: + pass + try: + kappa_sc = float(input_data["kappa_sc"]) + except KeyError: + pass + try: + s0_sc = float(input_data["s0_sc"]) + except KeyError: + pass + try: + order = int(input_data["order"]) + except KeyError: + pass + try: + integrator = input_data["integrator"] + except KeyError: + pass + try: + health_pres_min = float(input_data["health_pres_min"]) + except KeyError: + pass + try: + health_pres_max = float(input_data["health_pres_max"]) + except KeyError: + pass + + # param sanity check + allowed_integrators = ["rk4", "euler", "lsrk54", "lsrk144"] + if integrator not in allowed_integrators: + error_message = "Invalid time integrator: {}".format(integrator) + raise RuntimeError(error_message) + + if rank == 0: + print("#### Simluation control data: ####") + print(f"\tnviz = {nviz}") + print(f"\tnrestart = {nrestart}") + print(f"\tnhealth = {nhealth}") + print(f"\tnstatus = {nstatus}") + print(f"\tcurrent_dt = {current_dt}") + print(f"\tt_final = {t_final}") + print(f"\torder = {order}") + print(f"\tShock capturing parameters: alpha {alpha_sc}, " + f"s0 {s0_sc}, kappa {kappa_sc}") + print(f"\tTime integration {integrator}") + if log_dependent: + print("\tDependent variable logging is ON.") + else: + print("\tDependent variable logging is OFF.") + print("#### Simluation control data: ####") + + timestepper = rk4_step + if integrator == "euler": + timestepper = euler_step + if integrator == "lsrk54": + timestepper = lsrk54_step + if integrator == "lsrk144": + timestepper = lsrk144_step + + dim = 3 + vel_inflow = np.zeros(shape=(dim, )) + vel_outflow = np.zeros(shape=(dim, )) + + # working gas: CO2 # + # gamma = 1.289 + # MW=44.009 g/mol + # cp = 37.135 J/mol-K, + # rho= 1.977 kg/m^3 @298K + gamma_co2 = 1.289 + r_co2 = 8314.59/44.009 + + # background + # 100 Pa + # 298 K + # rho = 1.77619667e-3 kg/m^3 + # velocity = 0,0,0 + rho_bkrnd = 1.77619667e-3 + pres_bkrnd = 100 + temp_bkrnd = 298 + + # nozzle inflow # + # + # stagnation tempertuare 298 K + # stagnation pressure 1.5e Pa + # + # isentropic expansion based on the area ratios between the inlet (r=13e-3m) + # and the throat (r=6.3e-3) + # + # calculate the inlet Mach number from the area ratio + nozzle_inlet_radius = 13.0e-3 + nozzle_throat_radius = 6.3e-3 + nozzle_inlet_area = math.pi*nozzle_inlet_radius*nozzle_inlet_radius + nozzle_throat_area = math.pi*nozzle_throat_radius*nozzle_throat_radius + inlet_area_ratio = nozzle_inlet_area/nozzle_throat_area + + def get_mach_from_area_ratio(area_ratio, gamma, mach_guess=0.01): + error = 1.0e-8 + next_error = 1.0e8 + g = gamma + m0 = mach_guess + while next_error > error: + r_gas = ( + (((2/(g + 1) + ((g - 1)/(g + 1)*m0*m0))**(((g + 1)/(2*g - 2))))/m0 + - area_ratio) + ) + drdm = ( + (2*((2/(g + 1) + ((g - 1)/(g + 1)*m0*m0))**(((g + 1)/(2*g - 2)))) + / (2*g - 2)*(g - 1)/(2/(g + 1) + ((g - 1)/(g + 1)*m0*m0)) + - ((2/(g + 1) + ((g - 1)/(g + 1)*m0*m0))**(((g + 1)/(2*g - 2)))) + * m0**(-2)) + ) + m1 = m0 - r_gas/drdm + next_error = abs(r_gas) + m0 = m1 + + return m1 + + def get_isentropic_pressure(mach, p0, gamma): + pressure = (1. + (gamma - 1.)*0.5*math.pow(mach, 2)) + pressure = p0*math.pow(pressure, (-gamma / (gamma - 1.))) + return pressure + + def get_isentropic_temperature(mach, t0, gamma): + temperature = (1. + (gamma - 1.)*0.5*math.pow(mach, 2)) + temperature = t0*math.pow(temperature, -1.0) + return temperature + + inlet_mach = get_mach_from_area_ratio(area_ratio=inlet_area_ratio, + gamma=gamma_co2, + mach_guess=0.01) + # ramp the stagnation pressure + start_ramp_pres = 1000 + ramp_interval = 1.0e-3 + t_ramp_start = 1.0e-5 + pres_inflow = get_isentropic_pressure(mach=inlet_mach, + p0=start_ramp_pres, + gamma=gamma_co2) + temp_inflow = get_isentropic_temperature(mach=inlet_mach, + t0=298, + gamma=gamma_co2) + rho_inflow = pres_inflow / temp_inflow / r_co2 + end_ramp_pres = 150000 + pres_inflow_final = get_isentropic_pressure(mach=inlet_mach, + p0=end_ramp_pres, + gamma=gamma_co2) + vel_inflow[0] = inlet_mach * math.sqrt( + gamma_co2 * pres_inflow / rho_inflow) + + if rank == 0: + print(f"inlet Mach number {inlet_mach}") + print(f"inlet temperature {temp_inflow}") + print(f"inlet pressure {pres_inflow}") + print(f"final inlet pressure {pres_inflow_final}") + + mu = 1.e-5 + kappa = rho_bkrnd*mu/0.75 + transport_model = SimpleTransport(viscosity=mu, thermal_conductivity=kappa) + eos = IdealSingleGas( + gamma=gamma_co2, + gas_const=r_co2, + transport_model=transport_model + ) + bulk_init = PlanarDiscontinuity(dim=dim, disc_location=-.30, sigma=0.005, + temperature_left=temp_inflow, temperature_right=temp_bkrnd, + pressure_left=pres_inflow, pressure_right=pres_bkrnd, + velocity_left=vel_inflow, velocity_right=vel_outflow) + + # pressure ramp function + def inflow_ramp_pressure( + t, + start_p=start_ramp_pres, + final_p=end_ramp_pres, + ramp_interval=ramp_interval, + t_ramp_start=t_ramp_start + ): + return actx.np.where( + actx.np.greater(t, t_ramp_start), + actx.np.minimum( + final_p, + start_p + (t - t_ramp_start) / ramp_interval * (final_p - start_p)), + start_p) + + class IsentropicInflow: + def __init__(self, *, dim=1, direc=0, t0=298, p0=1e5, mach=0.01, p_fun=None): + + self._p0 = p0 + self._t0 = t0 + self._dim = dim + self._direc = direc + self._mach = mach + if p_fun is not None: + self._p_fun = p_fun + + def __call__(self, x_vec, *, time=0, eos, **kwargs): + + if self._p_fun is not None: + p0 = self._p_fun(time) + else: + p0 = self._p0 + t0 = self._t0 + + gamma = eos.gamma() + gas_const = eos.gas_const() + pressure = get_isentropic_pressure( + mach=self._mach, + p0=p0, + gamma=gamma + ) + temperature = get_isentropic_temperature( + mach=self._mach, + t0=t0, + gamma=gamma + ) + rho = pressure/temperature/gas_const + + velocity = np.zeros(self._dim, dtype=object) + velocity[self._direc] = self._mach*actx.np.sqrt(gamma*pressure/rho) + + mass = 0.0*x_vec[0] + rho + mom = velocity*mass + energy = (pressure/(gamma - 1.0)) + np.dot(mom, mom)/(2.0*mass) + return make_conserved( + dim=self._dim, + mass=mass, + momentum=mom, + energy=energy + ) + + inflow_init = IsentropicInflow( + dim=dim, + t0=298, + p0=start_ramp_pres, + mach=inlet_mach, + p_fun=inflow_ramp_pressure + ) + outflow_init = Uniform( + dim=dim, + rho=rho_bkrnd, + p=pres_bkrnd, + velocity=vel_outflow + ) + + inflow = PrescribedInviscidBoundary(fluid_solution_func=inflow_init) + outflow = PrescribedInviscidBoundary(fluid_solution_func=outflow_init) + wall = IsothermalNoSlipBoundary() + + boundaries = { + DTAG_BOUNDARY("Inflow"): inflow, + DTAG_BOUNDARY("Outflow"): outflow, + DTAG_BOUNDARY("Wall"): wall + } + + viz_path = "viz_data/" + vizname = viz_path + casename + restart_path = "restart_data/" + restart_pattern = ( + restart_path + "{cname}-{step:06d}-{rank:04d}.pkl" + ) + + if restart_filename: # read the grid from restart data + restart_filename = f"{restart_filename}-{rank:04d}.pkl" + + from mirgecom.restart import read_restart_data + restart_data = read_restart_data(actx, restart_filename) + current_step = restart_data["step"] + current_t = restart_data["t"] + local_mesh = restart_data["local_mesh"] + local_nelements = local_mesh.nelements + global_nelements = restart_data["global_nelements"] + restart_order = int(restart_data["order"]) + + assert comm.Get_size() == restart_data["num_parts"] + else: + boundary_tag_to_face = { + "Inflow": ["-x"], + "Outflow": ["+x"], + "Wall": ["-y", "+y", "-z", "+z"] + } + scale = np.power(weak_scale, 1/dim) + box_ll = 0 + box_ur = .5*scale + nel_1d = int(8*scale) + from meshmode.mesh.generation import generate_regular_rect_mesh + generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,) * dim, + b=(box_ur,) * dim, + nelements_per_axis=(nel_1d,) * dim, + boundary_tag_to_face=boundary_tag_to_face) + local_mesh, global_nelements = generate_and_distribute_mesh( + comm, + generate_mesh + ) + local_nelements = local_mesh.nelements + + if rank == 0: + logging.info("Making discretization") + + discr = EagerDGDiscretization(actx, + local_mesh, + order=order, + mpi_communicator=comm) + + nodes = thaw(actx, discr.nodes()) + + # initialize the sponge field + def gen_sponge(): + thickness = 0.15 + amplitude = 1.0/current_dt/25.0 + x0 = 0.05 + + return amplitude * actx.np.where( + actx.np.greater(nodes[0], x0), + zeros + ((nodes[0] - x0) / thickness) * ((nodes[0] - x0) / thickness), + zeros + 0.0, + ) + + zeros = 0 * nodes[0] + sponge_sigma = gen_sponge() + ref_state = bulk_init(x_vec=nodes, eos=eos, time=0.0) + + if restart_filename: + if rank == 0: + logging.info("Restarting soln.") + current_state = restart_data["state"] + if restart_order != order: + restart_discr = EagerDGDiscretization( + actx, + local_mesh, + order=restart_order, + mpi_communicator=comm) + from meshmode.discretization.connection import make_same_mesh_connection + connection = make_same_mesh_connection( + actx, + discr.discr_from_dd("vol"), + restart_discr.discr_from_dd("vol") + ) + restart_state = restart_data["state"] + current_state = connection(restart_state) + else: + if rank == 0: + logging.info("Initializing soln.") + # for Discontinuity initial conditions + current_state = bulk_init(x_vec=nodes, eos=eos, time=0.0) + # for uniform background initial condition + # current_state = bulk_init(nodes, eos=eos) + + vis_timer = None + log_cfl = LogUserQuantity(name="cfl", value=current_cfl) + + if logmgr: + logmgr_add_cl_device_info(logmgr, queue) + logmgr_set_time(logmgr, current_step, current_t) + logmgr.add_quantity(log_cfl, interval=nstatus) + + logmgr.add_watches([ + ("step.max", "step = {value}, "), + ("t_sim.max", "sim time: {value:1.6e} s, "), + ("cfl.max", "cfl = {value:1.4f}\n"), + ("t_step.max", "------- step walltime: {value:6g} s, "), + ("t_log.max", "log walltime: {value:6g} s\n") + ]) + + if log_dependent: + logmgr_add_many_discretization_quantities(logmgr, discr, dim, + extract_vars_for_logging, + units_for_logging) + logmgr.add_watches([ + ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), + ("max_pressure", "{value:1.9e})\n"), + ("min_temperature", "------- T (min, max) (K) = ({value:7g}, "), + ("max_temperature", "{value:7g})\n"), + ]) + + try: + logmgr.add_watches(["memory_usage.max"]) + except KeyError: + pass + + if use_profiling: + logmgr.add_watches(["pyopencl_array_time.max"]) + + vis_timer = IntervalTimer("t_vis", "Time spent visualizing") + logmgr.add_quantity(vis_timer) + + visualizer = make_visualizer(discr) + + initname = "pseudoY0" + eosname = eos.__class__.__name__ + init_message = make_init_message(dim=dim, + order=order, + nelements=local_nelements, + global_nelements=global_nelements, + dt=current_dt, + t_final=t_final, + nstatus=nstatus, + nviz=nviz, + cfl=current_cfl, + constant_cfl=constant_cfl, + initname=initname, + eosname=eosname, + casename=casename) + if rank == 0: + logger.info(init_message) + + def sponge(cv, cv_ref, sigma): + return (sigma*(cv_ref - cv)) + + def my_rhs(t, state): + return ( + ns_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) + + make_conserved( + dim, q=av_operator(discr, q=state.join(), boundaries=boundaries, + boundary_kwargs={"time": t, "eos": eos}, + alpha=alpha_sc, s0=s0_sc, kappa=kappa_sc) + ) + sponge(cv=state, cv_ref=ref_state, sigma=sponge_sigma) + ) + + def my_write_viz(step, t, dt, state, dv=None, tagged_cells=None, ts_field=None): + if dv is None: + dv = eos.dependent_vars(state) + if tagged_cells is None: + tagged_cells = smoothness_indicator(discr, state.mass, s0=s0_sc, + kappa=kappa_sc) + if ts_field is None: + ts_field, cfl, dt = my_get_timestep(t, dt, state) + + viz_fields = [("cv", state), + ("dv", dv), + ("sponge_sigma", gen_sponge()), + ("tagged_cells", tagged_cells), + ("dt" if constant_cfl else "cfl", ts_field)] + write_visfile(discr, viz_fields, visualizer, vizname=vizname, + step=step, t=t, overwrite=True) + + def my_write_restart(step, t, state): + restart_fname = restart_pattern.format(cname=casename, step=step, rank=rank) + if restart_fname != restart_filename: + restart_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": nparts + } + write_restart_file(actx, restart_data, restart_fname, comm) + + def my_health_check(dv): + health_error = False + if check_naninf_local(discr, "vol", dv.pressure): + health_error = True + logger.info(f"{rank=}: NANs/Infs in pressure data.") + + if check_range_local(discr, "vol", dv.pressure, + health_pres_min, health_pres_max): + health_error = True + logger.info(f"{rank=}: Pressure range violation.") + + return health_error + + def my_get_timestep(t, dt, state): + t_remaining = max(0, t_final - t) + if constant_cfl: + from mirgecom.viscous import get_viscous_timestep + ts_field = current_cfl * get_viscous_timestep(discr, eos=eos, cv=state) + from grudge.op import nodal_min + dt = nodal_min(discr, "vol", ts_field) + cfl = current_cfl + else: + from mirgecom.viscous import get_viscous_cfl + ts_field = get_viscous_cfl(discr, eos=eos, dt=dt, cv=state) + from grudge.op import nodal_max + cfl = nodal_max(discr, "vol", ts_field) + + return ts_field, cfl, min(t_remaining, dt) + + def my_pre_step(step, t, dt, state): + try: + dv = None + + if logmgr: + logmgr.tick_before() + + ts_field, cfl, dt = my_get_timestep(t, dt, state) + log_cfl.set_quantity(cfl) + + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) + + if do_health: + dv = eos.dependent_vars(state) + from mirgecom.simutil import allsync + health_errors = allsync(my_health_check(dv), comm, + op=MPI.LOR) + if health_errors: + if rank == 0: + logger.info("Fluid solution failed health check.") + raise MyRuntimeError("Failed simulation health check.") + + if do_restart: + my_write_restart(step=step, t=t, state=state) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + my_write_viz(step=step, t=t, dt=dt, state=state, dv=dv) + + except MyRuntimeError: + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + my_write_viz(step=step, t=t, dt=dt, state=state) + my_write_restart(step=step, t=t, state=state) + raise + + return state, dt + + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + + if rank == 0: + logging.info("Stepping.") + + current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + + (current_step, current_t, current_state) = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + pre_step_callback=my_pre_step, + post_step_callback=my_post_step, + state=current_state, dt=current_dt, + t_final=t_final, t=current_t, istep=current_step) + + # Dump the final data + if rank == 0: + logger.info("Checkpointing final state ...") + final_dv = eos.dependent_vars(current_state) + my_write_viz(step=current_step, t=current_t, dt=current_dt, state=current_state, + dv=final_dv) + my_write_restart(step=current_step, t=current_t, state=current_state) + + if logmgr: + logmgr.close() + elif use_profiling: + print(actx.tabulate_profiling_data()) + + exit() + + +if __name__ == "__main__": + import sys + + logging.basicConfig(format="%(message)s", level=logging.INFO) + + import argparse + parser = argparse.ArgumentParser( + description="MIRGE-Com Isentropic Nozzle Driver") + parser.add_argument("-r", "--restart_file", type=ascii, dest="restart_file", + nargs="?", action="store", help="simulation restart file") + parser.add_argument("-i", "--input_file", type=ascii, dest="input_file", + nargs="?", action="store", help="simulation config file") + parser.add_argument("-c", "--casename", type=ascii, dest="casename", + nargs="?", action="store", help="simulation case name") + parser.add_argument("--profile", action="store_true", default=False, + help="enable kernel profiling [OFF]") + parser.add_argument("--log", action="store_true", default=False, + help="enable logging profiling [ON]") + parser.add_argument("--lazy", action="store_true", default=False, + help="enable lazy evaluation [OFF]") + args = parser.parse_args() + + # for writing output + casename = "nozzle" + if args.casename: + print(f"Custom casename {args.casename}") + casename = args.casename.replace("'", "") + else: + print(f"Default casename {casename}") + + if args.profile: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext + + restart_filename = None + if args.restart_file: + restart_filename = (args.restart_file).replace("'", "") + print(f"Restarting from file: {restart_filename}") + + input_file = None + if args.input_file: + input_file = args.input_file.replace("'", "") + print(f"Reading user input from file: {input_file}") + else: + print("No user input file, using default values") + + print(f"Running {sys.argv[0]}\n") + main(restart_filename=restart_filename, use_profiling=args.profile, + use_logmgr=args.log, user_input_file=input_file, + actx_class=actx_class, casename=casename) + +# vim: foldmethod=marker From 172f5ef342efe99c31c225aa622bc5966d39c0e7 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Wed, 1 Sep 2021 13:32:03 -0700 Subject: [PATCH 310/385] Add quick temporary util to get boundary info for performance debugging --- mirgecom/simutil.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 5a792ae76..8e2106cc9 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -14,6 +14,7 @@ .. autofunction:: compare_fluid_solutions .. autofunction:: check_naninf_local .. autofunction:: check_range_local +.. autofunction:: boundary_report Mesh utilities -------------- @@ -262,7 +263,50 @@ def generate_and_distribute_mesh(comm, generate_mesh): return local_mesh, global_nelements +def boundary_report(discr, boundaries, outfile_name): + """Generate a report of the grid boundaries.""" + comm = discr.mpi_communicator + actx = discr._setup_actx + nproc = 1 + rank = 0 + if comm is not None: + nproc = comm.Get_size() + rank = comm.Get_rank() + local_header = f"nproc: {nproc}\nrank: {rank}\n" + from io import StringIO + local_report = StringIO(local_header) + local_report.seek(0, 2) + from meshmode.dof_array import thaw + for btag in boundaries: + boundary_discr = discr.discr_from_dd(btag) + nnodes = sum([grp.ndofs for grp in boundary_discr.groups]) + local_report.write(f"{btag}: {nnodes}\n") + + if nproc > 1: + from meshmode.mesh import BTAG_PARTITION + from grudge.trace_pair import connected_ranks + remote_ranks = connected_ranks(discr) + local_report.write(f"remote_ranks: {remote_ranks}\n") + rank_nodes = [] + for remote_rank in remote_ranks: + boundary_discr = discr.discr_from_dd(BTAG_PARTITION(remote_rank)) + nnodes = sum([grp.ndofs for grp in boundary_discr.groups]) + rank_nodes.append(nnodes) + local_report.write(f"nnodes_pb: {rank_nodes}\n") + + local_report.write("-----\n") + local_report.seek(0) + + for irank in range(nproc): + if irank == rank: + f = open(outfile_name, "a+") + f.write(local_report.read()) + f.close() + if comm is not None: + comm.barrier() + + def create_parallel_grid(comm, generate_grid): """Generate and distribute mesh compatibility interface.""" from warnings import warn From 116d768781c8877069673bf5b9fb02548dec7f36 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 1 Sep 2021 15:35:23 -0500 Subject: [PATCH 311/385] Placate flake8 --- mirgecom/simutil.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 8e2106cc9..e0003558f 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -263,10 +263,10 @@ def generate_and_distribute_mesh(comm, generate_mesh): return local_mesh, global_nelements + def boundary_report(discr, boundaries, outfile_name): """Generate a report of the grid boundaries.""" comm = discr.mpi_communicator - actx = discr._setup_actx nproc = 1 rank = 0 if comm is not None: @@ -277,7 +277,7 @@ def boundary_report(discr, boundaries, outfile_name): from io import StringIO local_report = StringIO(local_header) local_report.seek(0, 2) - from meshmode.dof_array import thaw + for btag in boundaries: boundary_discr = discr.discr_from_dd(btag) nnodes = sum([grp.ndofs for grp in boundary_discr.groups]) @@ -306,7 +306,7 @@ def boundary_report(discr, boundaries, outfile_name): if comm is not None: comm.barrier() - + def create_parallel_grid(comm, generate_grid): """Generate and distribute mesh compatibility interface.""" from warnings import warn From b30e38337a870cf700efb195d1f1366b6fdd0df3 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Sun, 12 Sep 2021 02:09:19 -0500 Subject: [PATCH 312/385] Delete bozzle from y1-production - causes CI fail. --- examples/bozzle.py | 863 --------------------------------------------- 1 file changed, 863 deletions(-) delete mode 100644 examples/bozzle.py diff --git a/examples/bozzle.py b/examples/bozzle.py deleted file mode 100644 index d903a9d87..000000000 --- a/examples/bozzle.py +++ /dev/null @@ -1,863 +0,0 @@ -"""Production-like (nozzle-like) case in a box.""" - -__copyright__ = """ -Copyright (C) 2020 University of Illinois Board of Trustees -""" - -__license__ = """ -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -""" -import os -import yaml -import logging -import numpy as np -import pyopencl as cl -import numpy.linalg as la # noqa -import pyopencl.array as cla # noqa -from functools import partial -import math - -from meshmode.dof_array import thaw -from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa -from grudge.dof_desc import DTAG_BOUNDARY -from grudge.eager import EagerDGDiscretization -from grudge.shortcuts import make_visualizer - -from meshmode.array_context import ( - PyOpenCLArrayContext, - SingleGridWorkBalancingPytatoArrayContext as PytatoPyOpenCLArrayContext -) -from mirgecom.profiling import PyOpenCLProfilingArrayContext - -from mirgecom.navierstokes import ns_operator -from mirgecom.fluid import make_conserved -from mirgecom.artificial_viscosity import (av_operator, smoothness_indicator) -from mirgecom.simutil import ( - check_step, - generate_and_distribute_mesh, - write_visfile, - check_naninf_local, - check_range_local, - get_sim_timestep -) -from mirgecom.restart import write_restart_file -from mirgecom.io import make_init_message -from mirgecom.mpi import mpi_entry_point -import pyopencl.tools as cl_tools -# from mirgecom.checkstate import compare_states -from mirgecom.integrators import (rk4_step, lsrk54_step, lsrk144_step, - euler_step) -from mirgecom.steppers import advance_state -from mirgecom.boundary import ( - PrescribedInviscidBoundary, - IsothermalNoSlipBoundary -) -from mirgecom.initializers import (Uniform, PlanarDiscontinuity) -from mirgecom.eos import IdealSingleGas -from mirgecom.transport import SimpleTransport - -from logpyle import IntervalTimer, set_dt -from mirgecom.euler import extract_vars_for_logging, units_for_logging -from mirgecom.logging_quantities import ( - initialize_logmgr, logmgr_add_many_discretization_quantities, - logmgr_add_cl_device_info, logmgr_set_time, LogUserQuantity, - set_sim_state -) - -logger = logging.getLogger(__name__) - - -class MyRuntimeError(RuntimeError): - """Simple exception to kill the simulation.""" - - pass - - -def get_pseudo_y0_mesh(): - """Generate or import a grid using `gmsh`. - - Input required: - data/pseudoY0.brep (for mesh gen) - -or- - data/pseudoY0.msh (read existing mesh) - - This routine will generate a new grid if it does - not find the grid file (data/pseudoY0.msh), but - note that if the grid is generated in millimeters, - then the solution initialization and BCs need to be - adjusted or the grid needs to be scaled up to meters - before being used with the current main driver in this - example. - """ - from meshmode.mesh.io import (read_gmsh, generate_gmsh, - ScriptWithFilesSource) - if os.path.exists("data/pseudoY1nozzle.msh") is False: - mesh = generate_gmsh(ScriptWithFilesSource( - """ - Merge "data/pseudoY1nozzle.brep"; - Mesh.CharacteristicLengthMin = 1; - Mesh.CharacteristicLengthMax = 10; - Mesh.ElementOrder = 2; - Mesh.CharacteristicLengthExtendFromBoundary = 0; - - // Inside and end surfaces of nozzle/scramjet - Field[1] = Distance; - Field[1].NNodesByEdge = 100; - Field[1].FacesList = {5,7,8,9,10}; - Field[2] = Threshold; - Field[2].IField = 1; - Field[2].LcMin = 1; - Field[2].LcMax = 10; - Field[2].DistMin = 0; - Field[2].DistMax = 20; - - // Edges separating surfaces with boundary layer - // refinement from those without - // (Seems to give a smoother transition) - Field[3] = Distance; - Field[3].NNodesByEdge = 100; - Field[3].EdgesList = {5,10,14,16}; - Field[4] = Threshold; - Field[4].IField = 3; - Field[4].LcMin = 1; - Field[4].LcMax = 10; - Field[4].DistMin = 0; - Field[4].DistMax = 20; - - // Min of the two sections above - Field[5] = Min; - Field[5].FieldsList = {2,4}; - - Background Field = 5; - """, ["data/pseudoY1nozzle.brep"]), 3, target_unit="MM") - else: - mesh = read_gmsh("data/pseudoY1nozzle.msh") - - return mesh - - -@mpi_entry_point -def main(ctx_factory=cl.create_some_context, restart_filename=None, - use_profiling=False, use_logmgr=False, user_input_file=None, - actx_class=PyOpenCLArrayContext, casename=None): - """Drive the Y0 nozzle example.""" - cl_ctx = ctx_factory() - - from mpi4py import MPI - comm = MPI.COMM_WORLD - rank = comm.Get_rank() - nparts = comm.Get_size() - - if casename is None: - casename = "mirgecom" - - # logging and profiling - logmgr = initialize_logmgr(use_logmgr, - filename=f"{casename}.sqlite", mode="wo", mpi_comm=comm) - - if use_profiling: - queue = cl.CommandQueue(cl_ctx, - properties=cl.command_queue_properties.PROFILING_ENABLE) - else: - queue = cl.CommandQueue(cl_ctx) - - actx = actx_class( - queue, - allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) - - # Most of these can be set by the user input file - - # default i/o junk frequencies - nviz = 100 - nrestart = 100 - nhealth = 100 - nstatus = 1 - log_dependent = 1 - - # default timestepping control - integrator = "rk4" - current_dt = 5.0e-8 - t_final = 5.0e-6 - current_cfl = 1.0 - current_t = 0 - constant_cfl = False - current_step = 0 - - # default health status bounds - health_pres_min = 1.0e-1 - health_pres_max = 2.0e6 - - # discretization and model control - order = 1 - alpha_sc = 0.5 - s0_sc = -5.0 - kappa_sc = 0.5 - weak_scale = 1 - - if user_input_file: - input_data = None - if rank == 0: - with open(user_input_file) as f: - input_data = yaml.load(f, Loader=yaml.FullLoader) - input_data = comm.bcast(input_data, root=0) - try: - weak_scale = float(input_data["wscale"]) - except KeyError: - pass - try: - nviz = int(input_data["nviz"]) - except KeyError: - pass - try: - nrestart = int(input_data["nrestart"]) - except KeyError: - pass - try: - nhealth = int(input_data["nhealth"]) - except KeyError: - pass - try: - nstatus = int(input_data["nstatus"]) - except KeyError: - pass - try: - log_dependent = int(input_data["logDependent"]) - except KeyError: - pass - try: - current_dt = float(input_data["current_dt"]) - except KeyError: - pass - try: - t_final = float(input_data["t_final"]) - except KeyError: - pass - try: - alpha_sc = float(input_data["alpha_sc"]) - except KeyError: - pass - try: - kappa_sc = float(input_data["kappa_sc"]) - except KeyError: - pass - try: - s0_sc = float(input_data["s0_sc"]) - except KeyError: - pass - try: - order = int(input_data["order"]) - except KeyError: - pass - try: - integrator = input_data["integrator"] - except KeyError: - pass - try: - health_pres_min = float(input_data["health_pres_min"]) - except KeyError: - pass - try: - health_pres_max = float(input_data["health_pres_max"]) - except KeyError: - pass - - # param sanity check - allowed_integrators = ["rk4", "euler", "lsrk54", "lsrk144"] - if integrator not in allowed_integrators: - error_message = "Invalid time integrator: {}".format(integrator) - raise RuntimeError(error_message) - - if rank == 0: - print("#### Simluation control data: ####") - print(f"\tnviz = {nviz}") - print(f"\tnrestart = {nrestart}") - print(f"\tnhealth = {nhealth}") - print(f"\tnstatus = {nstatus}") - print(f"\tcurrent_dt = {current_dt}") - print(f"\tt_final = {t_final}") - print(f"\torder = {order}") - print(f"\tShock capturing parameters: alpha {alpha_sc}, " - f"s0 {s0_sc}, kappa {kappa_sc}") - print(f"\tTime integration {integrator}") - if log_dependent: - print("\tDependent variable logging is ON.") - else: - print("\tDependent variable logging is OFF.") - print("#### Simluation control data: ####") - - timestepper = rk4_step - if integrator == "euler": - timestepper = euler_step - if integrator == "lsrk54": - timestepper = lsrk54_step - if integrator == "lsrk144": - timestepper = lsrk144_step - - dim = 3 - vel_inflow = np.zeros(shape=(dim, )) - vel_outflow = np.zeros(shape=(dim, )) - - # working gas: CO2 # - # gamma = 1.289 - # MW=44.009 g/mol - # cp = 37.135 J/mol-K, - # rho= 1.977 kg/m^3 @298K - gamma_co2 = 1.289 - r_co2 = 8314.59/44.009 - - # background - # 100 Pa - # 298 K - # rho = 1.77619667e-3 kg/m^3 - # velocity = 0,0,0 - rho_bkrnd = 1.77619667e-3 - pres_bkrnd = 100 - temp_bkrnd = 298 - - # nozzle inflow # - # - # stagnation tempertuare 298 K - # stagnation pressure 1.5e Pa - # - # isentropic expansion based on the area ratios between the inlet (r=13e-3m) - # and the throat (r=6.3e-3) - # - # calculate the inlet Mach number from the area ratio - nozzle_inlet_radius = 13.0e-3 - nozzle_throat_radius = 6.3e-3 - nozzle_inlet_area = math.pi*nozzle_inlet_radius*nozzle_inlet_radius - nozzle_throat_area = math.pi*nozzle_throat_radius*nozzle_throat_radius - inlet_area_ratio = nozzle_inlet_area/nozzle_throat_area - - def get_mach_from_area_ratio(area_ratio, gamma, mach_guess=0.01): - error = 1.0e-8 - next_error = 1.0e8 - g = gamma - m0 = mach_guess - while next_error > error: - r_gas = ( - (((2/(g + 1) + ((g - 1)/(g + 1)*m0*m0))**(((g + 1)/(2*g - 2))))/m0 - - area_ratio) - ) - drdm = ( - (2*((2/(g + 1) + ((g - 1)/(g + 1)*m0*m0))**(((g + 1)/(2*g - 2)))) - / (2*g - 2)*(g - 1)/(2/(g + 1) + ((g - 1)/(g + 1)*m0*m0)) - - ((2/(g + 1) + ((g - 1)/(g + 1)*m0*m0))**(((g + 1)/(2*g - 2)))) - * m0**(-2)) - ) - m1 = m0 - r_gas/drdm - next_error = abs(r_gas) - m0 = m1 - - return m1 - - def get_isentropic_pressure(mach, p0, gamma): - pressure = (1. + (gamma - 1.)*0.5*math.pow(mach, 2)) - pressure = p0*math.pow(pressure, (-gamma / (gamma - 1.))) - return pressure - - def get_isentropic_temperature(mach, t0, gamma): - temperature = (1. + (gamma - 1.)*0.5*math.pow(mach, 2)) - temperature = t0*math.pow(temperature, -1.0) - return temperature - - inlet_mach = get_mach_from_area_ratio(area_ratio=inlet_area_ratio, - gamma=gamma_co2, - mach_guess=0.01) - # ramp the stagnation pressure - start_ramp_pres = 1000 - ramp_interval = 1.0e-3 - t_ramp_start = 1.0e-5 - pres_inflow = get_isentropic_pressure(mach=inlet_mach, - p0=start_ramp_pres, - gamma=gamma_co2) - temp_inflow = get_isentropic_temperature(mach=inlet_mach, - t0=298, - gamma=gamma_co2) - rho_inflow = pres_inflow / temp_inflow / r_co2 - end_ramp_pres = 150000 - pres_inflow_final = get_isentropic_pressure(mach=inlet_mach, - p0=end_ramp_pres, - gamma=gamma_co2) - vel_inflow[0] = inlet_mach * math.sqrt( - gamma_co2 * pres_inflow / rho_inflow) - - if rank == 0: - print(f"inlet Mach number {inlet_mach}") - print(f"inlet temperature {temp_inflow}") - print(f"inlet pressure {pres_inflow}") - print(f"final inlet pressure {pres_inflow_final}") - - mu = 1.e-5 - kappa = rho_bkrnd*mu/0.75 - transport_model = SimpleTransport(viscosity=mu, thermal_conductivity=kappa) - eos = IdealSingleGas( - gamma=gamma_co2, - gas_const=r_co2, - transport_model=transport_model - ) - bulk_init = PlanarDiscontinuity(dim=dim, disc_location=-.30, sigma=0.005, - temperature_left=temp_inflow, temperature_right=temp_bkrnd, - pressure_left=pres_inflow, pressure_right=pres_bkrnd, - velocity_left=vel_inflow, velocity_right=vel_outflow) - - # pressure ramp function - def inflow_ramp_pressure( - t, - start_p=start_ramp_pres, - final_p=end_ramp_pres, - ramp_interval=ramp_interval, - t_ramp_start=t_ramp_start - ): - return actx.np.where( - actx.np.greater(t, t_ramp_start), - actx.np.minimum( - final_p, - start_p + (t - t_ramp_start) / ramp_interval * (final_p - start_p)), - start_p) - - class IsentropicInflow: - def __init__(self, *, dim=1, direc=0, t0=298, p0=1e5, mach=0.01, p_fun=None): - - self._p0 = p0 - self._t0 = t0 - self._dim = dim - self._direc = direc - self._mach = mach - if p_fun is not None: - self._p_fun = p_fun - - def __call__(self, x_vec, *, time=0, eos, **kwargs): - - if self._p_fun is not None: - p0 = self._p_fun(time) - else: - p0 = self._p0 - t0 = self._t0 - - gamma = eos.gamma() - gas_const = eos.gas_const() - pressure = get_isentropic_pressure( - mach=self._mach, - p0=p0, - gamma=gamma - ) - temperature = get_isentropic_temperature( - mach=self._mach, - t0=t0, - gamma=gamma - ) - rho = pressure/temperature/gas_const - - velocity = np.zeros(self._dim, dtype=object) - velocity[self._direc] = self._mach*actx.np.sqrt(gamma*pressure/rho) - - mass = 0.0*x_vec[0] + rho - mom = velocity*mass - energy = (pressure/(gamma - 1.0)) + np.dot(mom, mom)/(2.0*mass) - return make_conserved( - dim=self._dim, - mass=mass, - momentum=mom, - energy=energy - ) - - inflow_init = IsentropicInflow( - dim=dim, - t0=298, - p0=start_ramp_pres, - mach=inlet_mach, - p_fun=inflow_ramp_pressure - ) - outflow_init = Uniform( - dim=dim, - rho=rho_bkrnd, - p=pres_bkrnd, - velocity=vel_outflow - ) - - inflow = PrescribedInviscidBoundary(fluid_solution_func=inflow_init) - outflow = PrescribedInviscidBoundary(fluid_solution_func=outflow_init) - wall = IsothermalNoSlipBoundary() - - boundaries = { - DTAG_BOUNDARY("Inflow"): inflow, - DTAG_BOUNDARY("Outflow"): outflow, - DTAG_BOUNDARY("Wall"): wall - } - - viz_path = "viz_data/" - vizname = viz_path + casename - restart_path = "restart_data/" - restart_pattern = ( - restart_path + "{cname}-{step:06d}-{rank:04d}.pkl" - ) - - if restart_filename: # read the grid from restart data - restart_filename = f"{restart_filename}-{rank:04d}.pkl" - - from mirgecom.restart import read_restart_data - restart_data = read_restart_data(actx, restart_filename) - current_step = restart_data["step"] - current_t = restart_data["t"] - local_mesh = restart_data["local_mesh"] - local_nelements = local_mesh.nelements - global_nelements = restart_data["global_nelements"] - restart_order = int(restart_data["order"]) - - assert comm.Get_size() == restart_data["num_parts"] - else: - boundary_tag_to_face = { - "Inflow": ["-x"], - "Outflow": ["+x"], - "Wall": ["-y", "+y", "-z", "+z"] - } - scale = np.power(weak_scale, 1/dim) - box_ll = 0 - box_ur = .5*scale - nel_1d = int(8*scale) - from meshmode.mesh.generation import generate_regular_rect_mesh - generate_mesh = partial(generate_regular_rect_mesh, a=(box_ll,) * dim, - b=(box_ur,) * dim, - nelements_per_axis=(nel_1d,) * dim, - boundary_tag_to_face=boundary_tag_to_face) - local_mesh, global_nelements = generate_and_distribute_mesh( - comm, - generate_mesh - ) - local_nelements = local_mesh.nelements - - if rank == 0: - logging.info("Making discretization") - - discr = EagerDGDiscretization(actx, - local_mesh, - order=order, - mpi_communicator=comm) - - nodes = thaw(actx, discr.nodes()) - - # initialize the sponge field - def gen_sponge(): - thickness = 0.15 - amplitude = 1.0/current_dt/25.0 - x0 = 0.05 - - return amplitude * actx.np.where( - actx.np.greater(nodes[0], x0), - zeros + ((nodes[0] - x0) / thickness) * ((nodes[0] - x0) / thickness), - zeros + 0.0, - ) - - zeros = 0 * nodes[0] - sponge_sigma = gen_sponge() - ref_state = bulk_init(x_vec=nodes, eos=eos, time=0.0) - - if restart_filename: - if rank == 0: - logging.info("Restarting soln.") - current_state = restart_data["state"] - if restart_order != order: - restart_discr = EagerDGDiscretization( - actx, - local_mesh, - order=restart_order, - mpi_communicator=comm) - from meshmode.discretization.connection import make_same_mesh_connection - connection = make_same_mesh_connection( - actx, - discr.discr_from_dd("vol"), - restart_discr.discr_from_dd("vol") - ) - restart_state = restart_data["state"] - current_state = connection(restart_state) - else: - if rank == 0: - logging.info("Initializing soln.") - # for Discontinuity initial conditions - current_state = bulk_init(x_vec=nodes, eos=eos, time=0.0) - # for uniform background initial condition - # current_state = bulk_init(nodes, eos=eos) - - vis_timer = None - log_cfl = LogUserQuantity(name="cfl", value=current_cfl) - - if logmgr: - logmgr_add_cl_device_info(logmgr, queue) - logmgr_set_time(logmgr, current_step, current_t) - logmgr.add_quantity(log_cfl, interval=nstatus) - - logmgr.add_watches([ - ("step.max", "step = {value}, "), - ("t_sim.max", "sim time: {value:1.6e} s, "), - ("cfl.max", "cfl = {value:1.4f}\n"), - ("t_step.max", "------- step walltime: {value:6g} s, "), - ("t_log.max", "log walltime: {value:6g} s\n") - ]) - - if log_dependent: - logmgr_add_many_discretization_quantities(logmgr, discr, dim, - extract_vars_for_logging, - units_for_logging) - logmgr.add_watches([ - ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), - ("max_pressure", "{value:1.9e})\n"), - ("min_temperature", "------- T (min, max) (K) = ({value:7g}, "), - ("max_temperature", "{value:7g})\n"), - ]) - - try: - logmgr.add_watches(["memory_usage.max"]) - except KeyError: - pass - - if use_profiling: - logmgr.add_watches(["pyopencl_array_time.max"]) - - vis_timer = IntervalTimer("t_vis", "Time spent visualizing") - logmgr.add_quantity(vis_timer) - - visualizer = make_visualizer(discr) - - initname = "pseudoY0" - eosname = eos.__class__.__name__ - init_message = make_init_message(dim=dim, - order=order, - nelements=local_nelements, - global_nelements=global_nelements, - dt=current_dt, - t_final=t_final, - nstatus=nstatus, - nviz=nviz, - cfl=current_cfl, - constant_cfl=constant_cfl, - initname=initname, - eosname=eosname, - casename=casename) - if rank == 0: - logger.info(init_message) - - def sponge(cv, cv_ref, sigma): - return (sigma*(cv_ref - cv)) - - def my_rhs(t, state): - return ( - ns_operator(discr, cv=state, t=t, boundaries=boundaries, eos=eos) - + make_conserved( - dim, q=av_operator(discr, q=state.join(), boundaries=boundaries, - boundary_kwargs={"time": t, "eos": eos}, - alpha=alpha_sc, s0=s0_sc, kappa=kappa_sc) - ) + sponge(cv=state, cv_ref=ref_state, sigma=sponge_sigma) - ) - - def my_write_viz(step, t, dt, state, dv=None, tagged_cells=None, ts_field=None): - if dv is None: - dv = eos.dependent_vars(state) - if tagged_cells is None: - tagged_cells = smoothness_indicator(discr, state.mass, s0=s0_sc, - kappa=kappa_sc) - if ts_field is None: - ts_field, cfl, dt = my_get_timestep(t, dt, state) - - viz_fields = [("cv", state), - ("dv", dv), - ("sponge_sigma", gen_sponge()), - ("tagged_cells", tagged_cells), - ("dt" if constant_cfl else "cfl", ts_field)] - write_visfile(discr, viz_fields, visualizer, vizname=vizname, - step=step, t=t, overwrite=True) - - def my_write_restart(step, t, state): - restart_fname = restart_pattern.format(cname=casename, step=step, rank=rank) - if restart_fname != restart_filename: - restart_data = { - "local_mesh": local_mesh, - "state": state, - "t": t, - "step": step, - "order": order, - "global_nelements": global_nelements, - "num_parts": nparts - } - write_restart_file(actx, restart_data, restart_fname, comm) - - def my_health_check(dv): - health_error = False - if check_naninf_local(discr, "vol", dv.pressure): - health_error = True - logger.info(f"{rank=}: NANs/Infs in pressure data.") - - if check_range_local(discr, "vol", dv.pressure, - health_pres_min, health_pres_max): - health_error = True - logger.info(f"{rank=}: Pressure range violation.") - - return health_error - - def my_get_timestep(t, dt, state): - t_remaining = max(0, t_final - t) - if constant_cfl: - from mirgecom.viscous import get_viscous_timestep - ts_field = current_cfl * get_viscous_timestep(discr, eos=eos, cv=state) - from grudge.op import nodal_min - dt = nodal_min(discr, "vol", ts_field) - cfl = current_cfl - else: - from mirgecom.viscous import get_viscous_cfl - ts_field = get_viscous_cfl(discr, eos=eos, dt=dt, cv=state) - from grudge.op import nodal_max - cfl = nodal_max(discr, "vol", ts_field) - - return ts_field, cfl, min(t_remaining, dt) - - def my_pre_step(step, t, dt, state): - try: - dv = None - - if logmgr: - logmgr.tick_before() - - ts_field, cfl, dt = my_get_timestep(t, dt, state) - log_cfl.set_quantity(cfl) - - do_viz = check_step(step=step, interval=nviz) - do_restart = check_step(step=step, interval=nrestart) - do_health = check_step(step=step, interval=nhealth) - - if do_health: - dv = eos.dependent_vars(state) - from mirgecom.simutil import allsync - health_errors = allsync(my_health_check(dv), comm, - op=MPI.LOR) - if health_errors: - if rank == 0: - logger.info("Fluid solution failed health check.") - raise MyRuntimeError("Failed simulation health check.") - - if do_restart: - my_write_restart(step=step, t=t, state=state) - - if do_viz: - if dv is None: - dv = eos.dependent_vars(state) - my_write_viz(step=step, t=t, dt=dt, state=state, dv=dv) - - except MyRuntimeError: - if rank == 0: - logger.info("Errors detected; attempting graceful exit.") - my_write_viz(step=step, t=t, dt=dt, state=state) - my_write_restart(step=step, t=t, state=state) - raise - - return state, dt - - def my_post_step(step, t, dt, state): - # Logmgr needs to know about EOS, dt, dim? - # imo this is a design/scope flaw - if logmgr: - set_dt(logmgr, dt) - set_sim_state(logmgr, dim, state, eos) - logmgr.tick_after() - return state, dt - - if rank == 0: - logging.info("Stepping.") - - current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, - current_cfl, eos, t_final, constant_cfl) - - (current_step, current_t, current_state) = \ - advance_state(rhs=my_rhs, timestepper=timestepper, - pre_step_callback=my_pre_step, - post_step_callback=my_post_step, - state=current_state, dt=current_dt, - t_final=t_final, t=current_t, istep=current_step) - - # Dump the final data - if rank == 0: - logger.info("Checkpointing final state ...") - final_dv = eos.dependent_vars(current_state) - my_write_viz(step=current_step, t=current_t, dt=current_dt, state=current_state, - dv=final_dv) - my_write_restart(step=current_step, t=current_t, state=current_state) - - if logmgr: - logmgr.close() - elif use_profiling: - print(actx.tabulate_profiling_data()) - - exit() - - -if __name__ == "__main__": - import sys - - logging.basicConfig(format="%(message)s", level=logging.INFO) - - import argparse - parser = argparse.ArgumentParser( - description="MIRGE-Com Isentropic Nozzle Driver") - parser.add_argument("-r", "--restart_file", type=ascii, dest="restart_file", - nargs="?", action="store", help="simulation restart file") - parser.add_argument("-i", "--input_file", type=ascii, dest="input_file", - nargs="?", action="store", help="simulation config file") - parser.add_argument("-c", "--casename", type=ascii, dest="casename", - nargs="?", action="store", help="simulation case name") - parser.add_argument("--profile", action="store_true", default=False, - help="enable kernel profiling [OFF]") - parser.add_argument("--log", action="store_true", default=False, - help="enable logging profiling [ON]") - parser.add_argument("--lazy", action="store_true", default=False, - help="enable lazy evaluation [OFF]") - args = parser.parse_args() - - # for writing output - casename = "nozzle" - if args.casename: - print(f"Custom casename {args.casename}") - casename = args.casename.replace("'", "") - else: - print(f"Default casename {casename}") - - if args.profile: - if args.lazy: - raise ValueError("Can't use lazy and profiling together.") - actx_class = PyOpenCLProfilingArrayContext - else: - actx_class = PytatoPyOpenCLArrayContext if args.lazy \ - else PyOpenCLArrayContext - - restart_filename = None - if args.restart_file: - restart_filename = (args.restart_file).replace("'", "") - print(f"Restarting from file: {restart_filename}") - - input_file = None - if args.input_file: - input_file = args.input_file.replace("'", "") - print(f"Reading user input from file: {input_file}") - else: - print("No user input file, using default values") - - print(f"Running {sys.argv[0]}\n") - main(restart_filename=restart_filename, use_profiling=args.profile, - use_logmgr=args.log, user_input_file=input_file, - actx_class=actx_class, casename=casename) - -# vim: foldmethod=marker From 8c953dd6424b375f5959bdb175e04540574e76a1 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 17 Sep 2021 11:26:51 -0500 Subject: [PATCH 313/385] mix-enable adiabaticnoslip --- mirgecom/boundary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 18364b3d4..a088062e5 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -434,7 +434,7 @@ def exterior_soln(self, discr, cv, btag, **kwargs): # Form the external boundary solution with the new momentum return make_conserved(dim=dim, mass=int_cv.mass, energy=int_cv.energy, - momentum=ext_mom) + momentum=ext_mom, species_mass=int_cv.species_mass) def exterior_grad_q(self, nodes, nhat, grad_cv, **kwargs): """Get the exterior solution on the boundary.""" From fc11f40260f3dd73f570bf13f1b9a7e1b32cbe3e Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 20 Sep 2021 10:13:11 -0500 Subject: [PATCH 314/385] Switch-up production testing env temporarily --- .ci-support/production-testing-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 1adeba830..6d9ff38d5 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -15,7 +15,7 @@ set -x # patched by the incoming development. The following vars control the # production environment: # -# export PRODUCTION_BRANCH="" # The base production branch to be installed by emirge +export PRODUCTION_BRANCH="test-y1" # The base production branch to be installed by emirge # export PRODUCTION_FORK="" # The fork/home of production changes (if any) # # Multiple production drivers are supported. The user should provide a ':'-delimited From 9bb772f47dbfc632880912b8e1553c7e79f2b4d2 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 20 Sep 2021 10:29:15 -0500 Subject: [PATCH 315/385] Update production install cause we are testing production branch itself --- .ci-support/production-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh index 00b7f228c..d74c1142c 100644 --- a/.ci-support/production-install.sh +++ b/.ci-support/production-install.sh @@ -34,7 +34,7 @@ if [ -n "$DEVELOPMENT_BRANCH" ]; then fi DEVELOPMENT_BRANCH=${DEVELOPMENT_BRANCH:-"main"} DEVELOPMENT_FORK=${DEVELOPMENT_FORK:-"illinois-ceesd"} -PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"y1-production"} +PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"tesy-y1"} PRODUCTION_FORK=${PRODUCTION_FORK:-"illinois-ceesd"} echo "Production environment settings:" From 5ad71c26609f5b761443126245a978ae0dd52432 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 20 Sep 2021 10:46:50 -0500 Subject: [PATCH 316/385] Update production install cause we are testing production branch itself --- .ci-support/production-install.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh index d74c1142c..2440a4372 100644 --- a/.ci-support/production-install.sh +++ b/.ci-support/production-install.sh @@ -23,6 +23,7 @@ set -x if [ -n "$DEVELOPMENT_BRANCH" ]; then + echo "Preparing production environment for branch: ${DEVELOPMENT_BRANCH}." PRODUCTION_ENV_FILE="$1" if [ -e "$PRODUCTION_ENV_FILE" ]; then echo "Reading production configuration for ${DEVELOPMENT_BRANCH}." @@ -31,10 +32,13 @@ if [ -n "$DEVELOPMENT_BRANCH" ]; then echo "Using default production configuration for ${DEVELOPMENT_BRANCH}." echo "To customize, set up .ci-support/production-testing-env.sh." fi +else + echo "No development branch specified, probably main branch." fi + DEVELOPMENT_BRANCH=${DEVELOPMENT_BRANCH:-"main"} DEVELOPMENT_FORK=${DEVELOPMENT_FORK:-"illinois-ceesd"} -PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"tesy-y1"} +PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"test-y1"} PRODUCTION_FORK=${PRODUCTION_FORK:-"illinois-ceesd"} echo "Production environment settings:" From 4c81845fc8625a1428e8e7c1170b8f3b07dacd85 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 20 Sep 2021 11:06:49 -0500 Subject: [PATCH 317/385] Revert to usual env before merge with y1-production --- .ci-support/production-install.sh | 2 +- .ci-support/production-testing-env.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh index 2440a4372..57ee3a0c1 100644 --- a/.ci-support/production-install.sh +++ b/.ci-support/production-install.sh @@ -38,7 +38,7 @@ fi DEVELOPMENT_BRANCH=${DEVELOPMENT_BRANCH:-"main"} DEVELOPMENT_FORK=${DEVELOPMENT_FORK:-"illinois-ceesd"} -PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"test-y1"} +PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"y1-production"} PRODUCTION_FORK=${PRODUCTION_FORK:-"illinois-ceesd"} echo "Production environment settings:" diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 6d9ff38d5..1adeba830 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -15,7 +15,7 @@ set -x # patched by the incoming development. The following vars control the # production environment: # -export PRODUCTION_BRANCH="test-y1" # The base production branch to be installed by emirge +# export PRODUCTION_BRANCH="" # The base production branch to be installed by emirge # export PRODUCTION_FORK="" # The fork/home of production changes (if any) # # Multiple production drivers are supported. The user should provide a ':'-delimited From 6128d44659e160e22bce87ac66fe70420190f3d7 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 6 Oct 2021 11:35:07 -0500 Subject: [PATCH 318/385] Correct some mistakes in isothermal noslip imp --- mirgecom/boundary.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index a088062e5..5d7eb998a 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -462,8 +462,7 @@ def isothermal_noslip_pair(self, discr, btag, eos, cv, **kwargs): """Get the interior and exterior solution (*cv*) on the boundary.""" cv_minus = discr.project("vol", btag, cv) - # t_plus = self._wall_temp + 0*cv_minus.mass - t_plus = eos.temperature(cv_minus) + t_plus = self._wall_temp + 0*cv_minus.mass velocity_plus = -cv_minus.momentum / cv_minus.mass mass_frac_plus = cv_minus.species_mass / cv_minus.mass @@ -483,7 +482,7 @@ def isothermal_noslip_pair(self, discr, btag, eos, cv, **kwargs): def temperature_bc(self, nodes, cv, temperature, eos, **kwargs): """Get temperature value to weakly prescribe wall bc.""" - return 2*self._wall_temp - temperature + return 0*temperature + self._wall_temp class PrescribedViscousBoundary(FluidBC): From ed0ae0dbf899a9caebdd4326282a0a10dbb68193 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Wed, 6 Oct 2021 16:34:53 -0500 Subject: [PATCH 319/385] Use correct BC --- examples/poiseuille-mpi.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/examples/poiseuille-mpi.py b/examples/poiseuille-mpi.py index c063d7aeb..5fb1e6b26 100644 --- a/examples/poiseuille-mpi.py +++ b/examples/poiseuille-mpi.py @@ -52,7 +52,7 @@ from mirgecom.steppers import advance_state from mirgecom.boundary import ( PrescribedViscousBoundary, - IsothermalNoSlipBoundary + AdiabaticNoslipMovingBoundary ) from mirgecom.transport import SimpleTransport from mirgecom.eos import IdealSingleGas @@ -118,9 +118,9 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, # timestepping control timestepper = rk4_step - t_final = 1e-6 + t_final = 1e-7 current_cfl = 0.05 - current_dt = 1e-8 + current_dt = 1e-10 current_t = 0 constant_cfl = True current_step = 0 @@ -153,7 +153,10 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, global_nelements = restart_data["global_nelements"] assert restart_data["nparts"] == nparts else: # generate the grid from scratch - npts_axis = (50, 30) + n_refine = 5 + npts_x = 10 + npts_y = 6 * n_refine + npts_axis = (npts_x, npts_y) box_ll = (left_boundary_location, ybottom) box_ur = (right_boundary_location, ytop) generate_mesh = partial(_get_box_mesh, 2, a=box_ll, b=box_ur, n=npts_axis) @@ -162,7 +165,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, generate_mesh) local_nelements = local_mesh.nelements - order = 1 + order = 2 discr = EagerDGDiscretization( actx, local_mesh, order=order, mpi_communicator=comm ) @@ -226,8 +229,8 @@ def poiseuille_2d(x_vec, eos, cv=None, **kwargs): boundaries = {DTAG_BOUNDARY("-1"): PrescribedViscousBoundary(q_func=initializer), DTAG_BOUNDARY("+1"): PrescribedViscousBoundary(q_func=initializer), - DTAG_BOUNDARY("-2"): IsothermalNoSlipBoundary(), - DTAG_BOUNDARY("+2"): IsothermalNoSlipBoundary()} + DTAG_BOUNDARY("-2"): AdiabaticNoslipMovingBoundary(), + DTAG_BOUNDARY("+2"): AdiabaticNoslipMovingBoundary()} if rst_filename: current_t = restart_data["t"] @@ -311,7 +314,7 @@ def my_health_check(state, dv, component_errors): from mirgecom.simutil import allsync if allsync(check_range_local(discr, "vol", dv.pressure, 9.999e4, 1.00101e5), - comm, op=MPI.LOR): + comm, op=MPI.LOR): health_error = True from grudge.op import nodal_max, nodal_min p_min = nodal_min(discr, "vol", dv.pressure) @@ -330,7 +333,7 @@ def my_health_check(state, dv, component_errors): t_max = nodal_max(discr, "vol", dv.temperature) logger.info(f"Temperature range violation ({t_min=}, {t_max=})") - exittol = 10 + exittol = .1 if max(component_errors) > exittol: health_error = True if rank == 0: From 10724ec8b451af032545b7d98ed6e12540c38dd0 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 7 Oct 2021 19:54:15 -0500 Subject: [PATCH 320/385] Add hotplate example --- examples/hotplate-mpi.py | 469 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 469 insertions(+) create mode 100644 examples/hotplate-mpi.py diff --git a/examples/hotplate-mpi.py b/examples/hotplate-mpi.py new file mode 100644 index 000000000..61df85a37 --- /dev/null +++ b/examples/hotplate-mpi.py @@ -0,0 +1,469 @@ +"""Demonstrate a fluid between two hot plates in 2d.""" + +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" +import logging +import numpy as np +import pyopencl as cl +import pyopencl.tools as cl_tools +from functools import partial + +from meshmode.array_context import ( + PyOpenCLArrayContext, + PytatoPyOpenCLArrayContext +) +from mirgecom.profiling import PyOpenCLProfilingArrayContext +from meshmode.dof_array import thaw +from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa + +from grudge.eager import EagerDGDiscretization +from grudge.shortcuts import make_visualizer +from grudge.dof_desc import DTAG_BOUNDARY + +from mirgecom.fluid import make_conserved +from mirgecom.navierstokes import ns_operator +from mirgecom.simutil import get_sim_timestep + +from mirgecom.io import make_init_message +from mirgecom.mpi import mpi_entry_point +from mirgecom.integrators import rk4_step +from mirgecom.steppers import advance_state +from mirgecom.boundary import ( + PrescribedViscousBoundary, + IsothermalNoSlipBoundary +) +from mirgecom.transport import SimpleTransport +from mirgecom.eos import IdealSingleGas + +from logpyle import IntervalTimer, set_dt +from mirgecom.euler import extract_vars_for_logging, units_for_logging +from mirgecom.logging_quantities import ( + initialize_logmgr, + logmgr_add_many_discretization_quantities, + logmgr_add_device_name, + logmgr_add_device_memory_usage, + set_sim_state +) + + +logger = logging.getLogger(__name__) + + +class MyRuntimeError(RuntimeError): + """Simple exception to kill the simulation.""" + + pass + + +# Box grid generator widget lifted from @majosm and slightly bent +def _get_box_mesh(dim, a, b, n, t=None): + dim_names = ["x", "y", "z"] + bttf = {} + for i in range(dim): + bttf["-"+str(i+1)] = ["-"+dim_names[i]] + bttf["+"+str(i+1)] = ["+"+dim_names[i]] + from meshmode.mesh.generation import generate_regular_rect_mesh as gen + return gen(a=a, b=b, n=n, boundary_tag_to_face=bttf, mesh_type=t) + + +@mpi_entry_point +def main(ctx_factory=cl.create_some_context, use_logmgr=True, + use_leap=False, use_profiling=False, casename=None, + rst_filename=None, actx_class=PyOpenCLArrayContext): + """Drive the example.""" + cl_ctx = ctx_factory() + + if casename is None: + casename = "mirgecom" + + from mpi4py import MPI + comm = MPI.COMM_WORLD + rank = comm.Get_rank() + nparts = comm.Get_size() + + logmgr = initialize_logmgr(use_logmgr, + filename=f"{casename}.sqlite", mode="wu", mpi_comm=comm) + + if use_profiling: + queue = cl.CommandQueue( + cl_ctx, properties=cl.command_queue_properties.PROFILING_ENABLE) + else: + queue = cl.CommandQueue(cl_ctx) + + actx = actx_class( + queue, + allocator=cl_tools.MemoryPool(cl_tools.ImmediateAllocator(queue))) + + # timestepping control + timestepper = rk4_step + t_final = 1e-6 + current_cfl = .1 + current_dt = 1e-8 + current_t = 0 + constant_cfl = False + current_step = 0 + + # some i/o frequencies + nstatus = 1 + nviz = 1 + nrestart = 100 + nhealth = 1 + + # some geometry setup + dim = 2 + if dim != 2: + raise ValueError("This example must be run with dim = 2.") + left_boundary_location = 0 + right_boundary_location = 0.1 + bottom_boundary_location = 0 + top_boundary_location = .02 + rst_path = "restart_data/" + rst_pattern = ( + rst_path + "{cname}-{step:04d}-{rank:04d}.pkl" + ) + if rst_filename: # read the grid from restart data + rst_filename = f"{rst_filename}-{rank:04d}.pkl" + + from mirgecom.restart import read_restart_data + restart_data = read_restart_data(actx, rst_filename) + local_mesh = restart_data["local_mesh"] + local_nelements = local_mesh.nelements + global_nelements = restart_data["global_nelements"] + assert restart_data["nparts"] == nparts + else: # generate the grid from scratch + n_refine = 2 + npts_x = 6 * n_refine + npts_y = 4 * n_refine + npts_axis = (npts_x, npts_y) + box_ll = (left_boundary_location, bottom_boundary_location) + box_ur = (right_boundary_location, top_boundary_location) + generate_mesh = partial(_get_box_mesh, 2, a=box_ll, b=box_ur, n=npts_axis) + from mirgecom.simutil import generate_and_distribute_mesh + local_mesh, global_nelements = generate_and_distribute_mesh(comm, + generate_mesh) + local_nelements = local_mesh.nelements + + order = 1 + discr = EagerDGDiscretization( + actx, local_mesh, order=order, mpi_communicator=comm + ) + nodes = thaw(actx, discr.nodes()) + + if logmgr: + logmgr_add_device_name(logmgr, queue) + logmgr_add_device_memory_usage(logmgr, queue) + logmgr_add_many_discretization_quantities(logmgr, discr, dim, + extract_vars_for_logging, units_for_logging) + + logmgr.add_watches([ + ("step.max", "step = {value}, "), + ("t_sim.max", "sim time: {value:1.6e} s\n"), + ("min_pressure", "------- P (min, max) (Pa) = ({value:1.9e}, "), + ("max_pressure", "{value:1.9e})\n"), + ("min_temperature", "------- T (min, max) (K) = ({value:1.9e}, "), + ("max_temperature", "{value:1.9e})\n"), + ("t_step.max", "------- step walltime: {value:6g} s, "), + ("t_log.max", "log walltime: {value:6g} s") + ]) + + vis_timer = IntervalTimer("t_vis", "Time spent visualizing") + logmgr.add_quantity(vis_timer) + + mu = 1.0 + kappa = 1.0 + + top_boundary_temperature = 400 + bottom_boundary_temperature = 300 + + def tramp_2d(x_vec, eos, cv=None, **kwargs): + y = x_vec[1] + ones = 0*y + 1.0 + l_y = top_boundary_location - bottom_boundary_location + p0 = eos.gas_const() * bottom_boundary_temperature + delta_temp = top_boundary_temperature - bottom_boundary_temperature + dtdy = delta_temp / l_y + temperature_y = bottom_boundary_temperature + dtdy*y + mass = p0 / (eos.gas_const() * temperature_y) + e0 = p0 / (eos.gamma() - 1.0) + velocity = 0 * x_vec + energy = e0 * ones + momentum = mass * velocity + return make_conserved(2, mass=mass, energy=energy, momentum=momentum) + + initializer = tramp_2d + eos = IdealSingleGas(transport_model=SimpleTransport(viscosity=mu, + thermal_conductivity=kappa)) + exact = initializer(x_vec=nodes, eos=eos) + + boundaries = {DTAG_BOUNDARY("-1"): PrescribedViscousBoundary(q_func=initializer), + DTAG_BOUNDARY("+1"): PrescribedViscousBoundary(q_func=initializer), + DTAG_BOUNDARY("-2"): IsothermalNoSlipBoundary( + wall_temperature=bottom_boundary_temperature), + DTAG_BOUNDARY("+2"): IsothermalNoSlipBoundary( + wall_temperature=top_boundary_temperature)} + + if rst_filename: + current_t = restart_data["t"] + current_step = restart_data["step"] + current_state = restart_data["state"] + if logmgr: + from mirgecom.logging_quantities import logmgr_set_time + logmgr_set_time(logmgr, current_step, current_t) + else: + # Set the current state from time 0 + current_state = exact + + vis_timer = None + + visualizer = make_visualizer(discr, order) + + eosname = eos.__class__.__name__ + init_message = make_init_message(dim=dim, order=order, + nelements=local_nelements, + global_nelements=global_nelements, + dt=current_dt, t_final=t_final, nstatus=nstatus, + nviz=nviz, cfl=current_cfl, + constant_cfl=constant_cfl, initname=casename, + eosname=eosname, casename=casename) + if rank == 0: + logger.info(init_message) + + def my_write_status(step, t, dt, dv, state, component_errors): + from grudge.op import nodal_min, nodal_max + p_min = nodal_min(discr, "vol", dv.pressure) + p_max = nodal_max(discr, "vol", dv.pressure) + t_min = nodal_min(discr, "vol", dv.temperature) + t_max = nodal_max(discr, "vol", dv.temperature) + if constant_cfl: + cfl = current_cfl + else: + from mirgecom.viscous import get_viscous_cfl + cfl = nodal_max(discr, "vol", + get_viscous_cfl(discr, eos, dt, state)) + if rank == 0: + logger.info(f"Step: {step}, T: {t}, DT: {dt}, CFL: {cfl}\n" + f"----- Pressure({p_min}, {p_max})\n" + f"----- Temperature({t_min}, {t_max})\n" + "----- errors=" + + ", ".join("%.3g" % en for en in component_errors)) + + def my_write_viz(step, t, state, dv=None): + if dv is None: + dv = eos.dependent_vars(state) + resid = state - exact + viz_fields = [("cv", state), + ("dv", dv), + ("exact", exact), + ("resid", resid)] + + from mirgecom.simutil import write_visfile + write_visfile(discr, viz_fields, visualizer, vizname=casename, + step=step, t=t, overwrite=True) + + def my_write_restart(step, t, state): + rst_fname = rst_pattern.format(cname=casename, step=step, rank=rank) + if rst_fname != rst_filename: + rst_data = { + "local_mesh": local_mesh, + "state": state, + "t": t, + "step": step, + "order": order, + "global_nelements": global_nelements, + "num_parts": nparts + } + from mirgecom.restart import write_restart_file + write_restart_file(actx, rst_data, rst_fname, comm) + + def my_health_check(state, dv, component_errors): + health_error = False + from mirgecom.simutil import check_naninf_local, check_range_local + if check_naninf_local(discr, "vol", dv.pressure): + health_error = True + logger.info(f"{rank=}: NANs/Infs in pressure data.") + + from mirgecom.simutil import allsync + if allsync(check_range_local(discr, "vol", dv.pressure, 86129, 86131), + comm, op=MPI.LOR): + health_error = True + from grudge.op import nodal_max, nodal_min + p_min = nodal_min(discr, "vol", dv.pressure) + p_max = nodal_max(discr, "vol", dv.pressure) + logger.info(f"Pressure range violation ({p_min=}, {p_max=})") + + if check_naninf_local(discr, "vol", dv.temperature): + health_error = True + logger.info(f"{rank=}: NANs/INFs in temperature data.") + + if allsync(check_range_local(discr, "vol", dv.temperature, 299, 401), + comm, op=MPI.LOR): + health_error = True + from grudge.op import nodal_max, nodal_min + t_min = nodal_min(discr, "vol", dv.temperature) + t_max = nodal_max(discr, "vol", dv.temperature) + logger.info(f"Temperature range violation ({t_min=}, {t_max=})") + + exittol = .1 + if max(component_errors) > exittol: + health_error = True + if rank == 0: + logger.info("Solution diverged from exact soln.") + + return health_error + + def my_pre_step(step, t, dt, state): + try: + dv = None + component_errors = None + + if logmgr: + logmgr.tick_before() + + from mirgecom.simutil import check_step + do_viz = check_step(step=step, interval=nviz) + do_restart = check_step(step=step, interval=nrestart) + do_health = check_step(step=step, interval=nhealth) + do_status = check_step(step=step, interval=nstatus) + + if do_health: + dv = eos.dependent_vars(state) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + from mirgecom.simutil import allsync + health_errors = allsync( + my_health_check(state, dv, component_errors), comm, + op=MPI.LOR + ) + if health_errors: + if rank == 0: + logger.info("Fluid solution failed health check.") + raise MyRuntimeError("Failed simulation health check.") + + if do_restart: + my_write_restart(step=step, t=t, state=state) + + if do_viz: + if dv is None: + dv = eos.dependent_vars(state) + my_write_viz(step=step, t=t, state=state, dv=dv) + + dt = get_sim_timestep(discr, state, t, dt, current_cfl, eos, + t_final, constant_cfl) + + if do_status: # needed because logging fails to make output + if dv is None: + dv = eos.dependent_vars(state) + if component_errors is None: + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, state, exact) + my_write_status(step=step, t=t, dt=dt, dv=dv, state=state, + component_errors=component_errors) + + except MyRuntimeError: + if rank == 0: + logger.info("Errors detected; attempting graceful exit.") + my_write_viz(step=step, t=t, state=state) + my_write_restart(step=step, t=t, state=state) + raise + + return state, dt + + def my_post_step(step, t, dt, state): + # Logmgr needs to know about EOS, dt, dim? + # imo this is a design/scope flaw + if logmgr: + set_dt(logmgr, dt) + set_sim_state(logmgr, dim, state, eos) + logmgr.tick_after() + return state, dt + + def my_rhs(t, state): + return ns_operator(discr, eos=eos, boundaries=boundaries, cv=state, t=t) + + current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + + current_step, current_t, current_state = \ + advance_state(rhs=my_rhs, timestepper=timestepper, + pre_step_callback=my_pre_step, + post_step_callback=my_post_step, dt=current_dt, + state=current_state, t=current_t, t_final=t_final) + + # Dump the final data + if rank == 0: + logger.info("Checkpointing final state ...") + final_dv = eos.dependent_vars(current_state) + final_dt = get_sim_timestep(discr, current_state, current_t, current_dt, + current_cfl, eos, t_final, constant_cfl) + from mirgecom.simutil import compare_fluid_solutions + component_errors = compare_fluid_solutions(discr, current_state, exact) + + my_write_viz(step=current_step, t=current_t, state=current_state, dv=final_dv) + my_write_restart(step=current_step, t=current_t, state=current_state) + my_write_status(step=current_step, t=current_t, dt=final_dt, dv=final_dv, + state=current_state, component_errors=component_errors) + + if logmgr: + logmgr.close() + elif use_profiling: + print(actx.tabulate_profiling_data()) + + finish_tol = 1e-16 + assert np.abs(current_t - t_final) < finish_tol + + +if __name__ == "__main__": + import argparse + casename = "hotplate" + parser = argparse.ArgumentParser(description=f"MIRGE-Com Example: {casename}") + parser.add_argument("--lazy", action="store_true", + help="switch to a lazy computation mode") + parser.add_argument("--profiling", action="store_true", + help="turn on detailed performance profiling") + parser.add_argument("--log", action="store_true", default=True, + help="turn on logging") + parser.add_argument("--leap", action="store_true", + help="use leap timestepper") + parser.add_argument("--restart_file", help="root name of restart file") + parser.add_argument("--casename", help="casename to use for i/o") + args = parser.parse_args() + if args.profiling: + if args.lazy: + raise ValueError("Can't use lazy and profiling together.") + actx_class = PyOpenCLProfilingArrayContext + else: + actx_class = PytatoPyOpenCLArrayContext if args.lazy \ + else PyOpenCLArrayContext + + logging.basicConfig(format="%(message)s", level=logging.INFO) + if args.casename: + casename = args.casename + rst_filename = None + if args.restart_file: + rst_filename = args.restart_file + + main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling, + casename=casename, rst_filename=rst_filename, actx_class=actx_class) + +# vim: foldmethod=marker From 60a7b86e90ebc5a515eb7ea16e3a6ccba2a0666f Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 11 Oct 2021 13:30:21 -0500 Subject: [PATCH 321/385] Fix Poiseuille test boundary issue --- test/test_navierstokes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_navierstokes.py b/test/test_navierstokes.py index 10ef64829..0f2e18ba5 100644 --- a/test/test_navierstokes.py +++ b/test/test_navierstokes.py @@ -45,7 +45,7 @@ from mirgecom.boundary import ( DummyBoundary, PrescribedViscousBoundary, - IsothermalNoSlipBoundary + AdiabaticNoslipMovingBoundary ) from mirgecom.eos import IdealSingleGas from mirgecom.transport import SimpleTransport @@ -286,8 +286,8 @@ def poiseuille_2d(x_vec, eos, cv=None, **kwargs): boundaries = { DTAG_BOUNDARY("-1"): PrescribedViscousBoundary(q_func=initializer), DTAG_BOUNDARY("+1"): PrescribedViscousBoundary(q_func=initializer), - DTAG_BOUNDARY("-2"): IsothermalNoSlipBoundary(), - DTAG_BOUNDARY("+2"): IsothermalNoSlipBoundary()} + DTAG_BOUNDARY("-2"): AdiabaticNoslipMovingBoundary(), + DTAG_BOUNDARY("+2"): AdiabaticNoslipMovingBoundary()} ns_rhs = ns_operator(discr, eos=eos, boundaries=boundaries, cv=cv_input, t=0.0) From 6e4cb1d307b674bfe8865d06e56c8456d6a91069 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 15 Oct 2021 16:32:52 -0500 Subject: [PATCH 322/385] Switch doublemach example back to Euler. --- examples/doublemach-mpi.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index 4cbf50efd..7a4968027 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -43,7 +43,7 @@ from grudge.shortcuts import make_visualizer -from mirgecom.navierstokes import ns_operator +from mirgecom.euler import euler_operator from mirgecom.artificial_viscosity import ( av_operator, smoothness_indicator @@ -55,7 +55,7 @@ from mirgecom.steppers import advance_state from mirgecom.boundary import ( AdiabaticNoslipMovingBoundary, - PrescribedBoundary + PrescribedInviscidBoundary ) from mirgecom.initializers import DoubleMachReflection from mirgecom.eos import IdealSingleGas @@ -227,9 +227,12 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, initializer = DoubleMachReflection() boundaries = { - DTAG_BOUNDARY("ic1"): PrescribedBoundary(initializer), - DTAG_BOUNDARY("ic2"): PrescribedBoundary(initializer), - DTAG_BOUNDARY("ic3"): PrescribedBoundary(initializer), + DTAG_BOUNDARY("ic1"): + PrescribedInviscidBoundary(fluid_solution_func=initializer), + DTAG_BOUNDARY("ic2"): + PrescribedInviscidBoundary(fluid_solution_func=initializer), + DTAG_BOUNDARY("ic3"): + PrescribedInviscidBoundary(fluid_solution_func=initializer), DTAG_BOUNDARY("wall"): AdiabaticNoslipMovingBoundary(), DTAG_BOUNDARY("out"): AdiabaticNoslipMovingBoundary(), } @@ -386,8 +389,8 @@ def my_post_step(step, t, dt, state): return state, dt def my_rhs(t, state): - return ns_operator( - discr, cv=state, t=t, boundaries=boundaries, eos=eos + return euler_operator( + discr, cv=state, time=t, boundaries=boundaries, eos=eos ) + make_conserved(dim, q=av_operator( discr, q=state.join(), boundaries=boundaries, boundary_kwargs={"time": t, "eos": eos}, alpha=alpha, From ca1367454f804e0e2fe8708a8c88fbb5d379287e Mon Sep 17 00:00:00 2001 From: Matthew Smith Date: Fri, 24 Sep 2021 16:38:44 -0500 Subject: [PATCH 323/385] account for the fact that nodal reductions now return device scalars --- examples/autoignition-mpi.py | 4 +-- examples/heat-source-mpi.py | 2 +- examples/vortex-mpi.py | 6 ++-- examples/wave-mpi.py | 8 ++--- examples/wave.py | 6 ++-- mirgecom/logging_quantities.py | 5 ++- mirgecom/simutil.py | 16 +++++---- test/test_bc.py | 9 ++--- test/test_diffusion.py | 10 +++--- test/test_eos.py | 61 +++++++++++++++++++++------------- test/test_euler.py | 34 +++++++++++-------- test/test_filter.py | 6 ++-- test/test_fluid.py | 22 ++++++++---- test/test_init.py | 38 +++++++++++++-------- test/test_inviscid.py | 26 ++++++++++----- test/test_lazy.py | 4 ++- test/test_operators.py | 11 +++--- test/test_restart.py | 2 +- test/test_simutil.py | 2 +- test/test_symbolic.py | 2 +- test/test_viscous.py | 47 +++++++++++++++----------- test/test_wave.py | 9 +++-- 22 files changed, 201 insertions(+), 129 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 5ec03576f..c64ccb810 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -380,13 +380,13 @@ def my_get_timestep(t, dt, state): from mirgecom.inviscid import get_inviscid_timestep ts_field = current_cfl * get_inviscid_timestep(discr, eos=eos, cv=state) from grudge.op import nodal_min - dt = nodal_min(discr, "vol", ts_field) + dt = actx.to_numpy(nodal_min(discr, "vol", ts_field))[()] cfl = current_cfl else: from mirgecom.inviscid import get_inviscid_cfl ts_field = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state) from grudge.op import nodal_max - cfl = nodal_max(discr, "vol", ts_field) + cfl = actx.to_numpy(nodal_max(discr, "vol", ts_field))[()] return ts_field, cfl, min(t_remaining, dt) diff --git a/examples/heat-source-mpi.py b/examples/heat-source-mpi.py index deeee6e1f..30e909112 100644 --- a/examples/heat-source-mpi.py +++ b/examples/heat-source-mpi.py @@ -182,7 +182,7 @@ def rhs(t, u): if logmgr: set_dt(logmgr, dt) logmgr.tick_after() - final_answer = discr.norm(u, np.inf) + final_answer = actx.to_numpy(discr.norm(u, np.inf)) resid = abs(final_answer - 0.00020620711665201585) if resid > 1e-15: raise ValueError(f"Run did not produce the expected result {resid=}") diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index b31e28cf4..fc5071029 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -227,8 +227,10 @@ def my_write_status(state, component_errors, cfl=None): else: from grudge.op import nodal_max from mirgecom.inviscid import get_inviscid_cfl - cfl = nodal_max(discr, "vol", - get_inviscid_cfl(discr, eos, current_dt, cv=state)) + cfl = actx.to_numpy( + nodal_max( + discr, "vol", + get_inviscid_cfl(discr, eos, current_dt, cv=state)))[()] if rank == 0: logger.info( f"------ {cfl=}\n" diff --git a/examples/wave-mpi.py b/examples/wave-mpi.py index 267ec9d9f..f7ba0f662 100644 --- a/examples/wave-mpi.py +++ b/examples/wave-mpi.py @@ -138,10 +138,10 @@ def main(snapshot_pattern="wave-mpi-{step:04d}-{rank:04d}.pkl", restart_step=Non current_cfl = 0.485 wave_speed = 1.0 from grudge.dt_utils import characteristic_lengthscales - dt = current_cfl * characteristic_lengthscales(actx, discr) / wave_speed + nodal_dt = characteristic_lengthscales(actx, discr) / wave_speed from grudge.op import nodal_min - dt = nodal_min(discr, "vol", dt) + dt = actx.to_numpy(current_cfl * nodal_min(discr, "vol", nodal_dt))[()] t_final = 1 @@ -217,7 +217,7 @@ def rhs(t, w): ) if istep % 10 == 0: - print(istep, t, discr.norm(fields[0])) + print(istep, t, actx.to_numpy(discr.norm(fields[0]))) vis.write_parallel_vtk_file( comm, "fld-wave-mpi-%03d-%04d.vtu" % (rank, istep), @@ -237,7 +237,7 @@ def rhs(t, w): set_dt(logmgr, dt) logmgr.tick_after() - final_soln = discr.norm(fields[0]) + final_soln = actx.to_numpy(discr.norm(fields[0])) assert np.abs(final_soln - 0.04409852463947439) < 1e-14 diff --git a/examples/wave.py b/examples/wave.py index b652b42ae..27d922a40 100644 --- a/examples/wave.py +++ b/examples/wave.py @@ -106,9 +106,9 @@ def main(use_profiling=False, use_logmgr=False, lazy_eval: bool = False): current_cfl = 0.485 wave_speed = 1.0 from grudge.dt_utils import characteristic_lengthscales - dt = current_cfl * characteristic_lengthscales(actx, discr) / wave_speed + nodal_dt = characteristic_lengthscales(actx, discr) / wave_speed from grudge.op import nodal_min - dt = nodal_min(discr, "vol", dt) + dt = actx.to_numpy(current_cfl * nodal_min(discr, "vol", nodal_dt))[()] print("%d elements" % mesh.nelements) @@ -152,7 +152,7 @@ def rhs(t, w): if istep % 10 == 0: if use_profiling: print(actx.tabulate_profiling_data()) - print(istep, t, discr.norm(fields[0], np.inf)) + print(istep, t, actx.to_numpy(discr.norm(fields[0], np.inf))) vis.write_vtk_file("fld-wave-%04d.vtu" % istep, [ ("u", fields[0]), diff --git a/mirgecom/logging_quantities.py b/mirgecom/logging_quantities.py index 5e94e1965..40dd2161f 100644 --- a/mirgecom/logging_quantities.py +++ b/mirgecom/logging_quantities.py @@ -42,6 +42,7 @@ from logpyle import (LogQuantity, PostLogQuantity, LogManager, MultiPostLogQuantity, add_run_info, add_general_quantities, add_simulation_quantities) +from arraycontext.container import get_container_context_recursively from meshmode.array_context import PyOpenCLArrayContext from meshmode.discretization import Discretization import pyopencl as cl @@ -279,10 +280,12 @@ def __call__(self): quantity = self.state_vars[self.quantity] + actx = get_container_context_recursively(quantity) + if self.axis is not None: # e.g. momentum quantity = quantity[self.axis] - return self._discr_reduction(quantity) + return actx.to_numpy(self._discr_reduction(quantity))[()] # }}} diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index d94cd3916..45f1cf6e1 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -119,10 +119,10 @@ def get_sim_timestep(discr, state, t, dt, cfl, eos, if constant_cfl: from mirgecom.viscous import get_viscous_timestep from grudge.op import nodal_min - mydt = cfl * nodal_min( - discr, "vol", - get_viscous_timestep(discr=discr, eos=eos, cv=state) - ) + mydt = state.array_context.to_numpy( + cfl * nodal_min( + discr, "vol", + get_viscous_timestep(discr=discr, eos=eos, cv=state)))[()] return min(t_remaining, mydt) @@ -269,9 +269,10 @@ def allsync(local_values, comm=None, op=None): def check_range_local(discr, dd, field, min_value, max_value): """Check for any negative values.""" + actx = field.array_context return ( - op.nodal_min_loc(discr, dd, field) < min_value - or op.nodal_max_loc(discr, dd, field) > max_value + actx.to_numpy(op.nodal_min_loc(discr, dd, field)) < min_value + or actx.to_numpy(op.nodal_max_loc(discr, dd, field)) > max_value ) @@ -288,8 +289,9 @@ def compare_fluid_solutions(discr, red_state, blue_state): .. note:: This is a collective routine and must be called by all MPI ranks. """ + actx = red_state.array_context resid = red_state - blue_state - return [discr.norm(v, np.inf) for v in resid.join()] + return [actx.to_numpy(discr.norm(v, np.inf)) for v in resid.join()] def generate_and_distribute_mesh(comm, generate_mesh): diff --git a/test/test_bc.py b/test/test_bc.py index 3ef0894c9..133edc95e 100644 --- a/test/test_bc.py +++ b/test/test_bc.py @@ -83,8 +83,9 @@ def test_slipwall_identity(actx_factory, dim): wall = AdiabaticSlipBoundary() uniform_state = initializer(nodes) - from functools import partial - bnd_norm = partial(discr.norm, p=np.inf, dd=BTAG_ALL) + + def bnd_norm(vec): + return actx.to_numpy(discr.norm(vec, p=np.inf, dd=BTAG_ALL)) bnd_pair = wall.boundary_pair(discr, btag=BTAG_ALL, eos=eos, cv=uniform_state) @@ -135,8 +136,8 @@ def test_slipwall_flux(actx_factory, dim, order): nhat = thaw(actx, discr.normal(BTAG_ALL)) h = 1.0 / nel_1d - from functools import partial - bnd_norm = partial(discr.norm, p=np.inf, dd=BTAG_ALL) + def bnd_norm(vec): + return actx.to_numpy(discr.norm(vec, p=np.inf, dd=BTAG_ALL)) logger.info(f"Number of {dim}d elems: {mesh.nelements}") # for velocities in each direction diff --git a/test/test_diffusion.py b/test/test_diffusion.py index e66b06293..7a74785d4 100644 --- a/test/test_diffusion.py +++ b/test/test_diffusion.py @@ -301,7 +301,7 @@ def get_rhs(t, u): expected_u = sym_eval(p.sym_u, t) - rel_linf_err = ( + rel_linf_err = actx.to_numpy( discr.norm(u - expected_u, np.inf) / discr.norm(expected_u, np.inf)) eoc_rec.add_data_point(1./n, rel_linf_err) @@ -385,7 +385,7 @@ def get_rhs(t, u): ("rhs", rhs), ]) - linf_err = discr.norm(rhs, np.inf) + linf_err = actx.to_numpy(discr.norm(rhs, np.inf)) assert(linf_err < 1e-11) # Now check stability @@ -415,7 +415,7 @@ def get_rhs(t, u): ("u_steady", u_steady), ]) - linf_diff = discr.norm(u - u_steady, np.inf) + linf_diff = actx.to_numpy(discr.norm(u - u_steady, np.inf)) assert linf_diff < 0.1 @@ -536,7 +536,7 @@ def sym_eval(expr): assert isinstance(diffusion_u1, DOFArray) expected_diffusion_u1 = sym_eval(sym_diffusion_u1) - rel_linf_err = ( + rel_linf_err = actx.to_numpy( discr.norm(diffusion_u1 - expected_diffusion_u1, np.inf) / discr.norm(expected_diffusion_u1, np.inf)) assert rel_linf_err < 1.e-5 @@ -556,7 +556,7 @@ def sym_eval(expr): sym_eval(sym_diffusion_u1), sym_eval(sym_diffusion_u2) ]) - rel_linf_err = ( + rel_linf_err = actx.to_numpy( discr.norm(diffusion_u_vector - expected_diffusion_u_vector, np.inf) / discr.norm(expected_diffusion_u_vector, np.inf)) assert rel_linf_err < 1.e-5 diff --git a/test/test_eos.py b/test/test_eos.py index 7c90c87c9..841cdbac4 100644 --- a/test/test_eos.py +++ b/test/test_eos.py @@ -149,18 +149,21 @@ def test_pyrometheus_mechanisms(ctx_factory, mechname, rate_tol, y0): print(f"can_omega = {can_omega}") print(f"prom_omega = {prom_omega}") - assert discr.norm((prom_c - can_c) / can_c, np.inf) < 1e-14 - assert discr.norm((prom_t - can_t) / can_t, np.inf) < 1e-14 - assert discr.norm((prom_rho - can_rho) / can_rho, np.inf) < 1e-14 - assert discr.norm((prom_p - can_p) / can_p, np.inf) < 1e-14 - assert discr.norm((prom_e - can_e) / can_e, np.inf) < 1e-6 - assert discr.norm((prom_k - can_k) / can_k, np.inf) < 1e-10 + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + assert inf_norm((prom_c - can_c) / can_c) < 1e-14 + assert inf_norm((prom_t - can_t) / can_t) < 1e-14 + assert inf_norm((prom_rho - can_rho) / can_rho) < 1e-14 + assert inf_norm((prom_p - can_p) / can_p) < 1e-14 + assert inf_norm((prom_e - can_e) / can_e) < 1e-6 + assert inf_norm((prom_k - can_k) / can_k) < 1e-10 # Pyro chem test comparisons for i, rate in enumerate(can_r): - assert discr.norm((prom_r[i] - rate), np.inf) < rate_tol + assert inf_norm(prom_r[i] - rate) < rate_tol for i, rate in enumerate(can_omega): - assert discr.norm((prom_omega[i] - rate), np.inf) < rate_tol + assert inf_norm(prom_omega[i] - rate) < rate_tol @pytest.mark.parametrize("mechname", ["uiuc", "sanDiego"]) @@ -246,11 +249,14 @@ def test_pyrometheus_eos(ctx_factory, mechname, dim, y0, vel): print(f"pyro_eos.temp = {temperature}") print(f"pyro_eos.e = {internal_energy}") + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + tol = 1e-14 - assert discr.norm((cv.mass - pyro_rho) / pyro_rho, np.inf) < tol - assert discr.norm((temperature - pyro_t) / pyro_t, np.inf) < tol - assert discr.norm((internal_energy - pyro_e) / pyro_e, np.inf) < tol - assert discr.norm((p - pyro_p) / pyro_p, np.inf) < tol + assert inf_norm((cv.mass - pyro_rho) / pyro_rho) < tol + assert inf_norm((temperature - pyro_t) / pyro_t) < tol + assert inf_norm((internal_energy - pyro_e) / pyro_e) < tol + assert inf_norm((p - pyro_p) / pyro_p) < tol @pytest.mark.parametrize(("mechname", "rate_tol"), @@ -345,24 +351,27 @@ def test_pyrometheus_kinetics(ctx_factory, mechname, rate_tol, y0): pyro_omega = pyro_obj.get_net_production_rates(rhoin, tin, yin) # Print + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + print(f"can_r = {can_r}") print(f"pyro_r = {pyro_r}") - abs_diff = discr.norm(pyro_r - can_r, np.inf) + abs_diff = inf_norm(pyro_r - can_r) if abs_diff > 1e-14: min_r = (np.abs(can_r)).min() if min_r > 0: - assert discr.norm((pyro_r - can_r) / can_r, np.inf) < rate_tol + assert inf_norm((pyro_r - can_r) / can_r) < rate_tol else: - assert discr.norm(pyro_r, np.inf) < rate_tol + assert inf_norm(pyro_r) < rate_tol print(f"can_omega = {can_omega}") print(f"pyro_omega = {pyro_omega}") for i, omega in enumerate(can_omega): omin = np.abs(omega).min() if omin > 1e-12: - assert discr.norm((pyro_omega[i] - omega) / omega, np.inf) < 1e-8 + assert inf_norm((pyro_omega[i] - omega) / omega) < 1e-8 else: - assert discr.norm(pyro_omega[i], np.inf) < 1e-12 + assert inf_norm(pyro_omega[i]) < 1e-12 @pytest.mark.parametrize("dim", [1, 2, 3]) @@ -398,16 +407,19 @@ def test_idealsingle_lump(ctx_factory, dim): eos = IdealSingleGas() cv = lump(nodes) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + p = eos.pressure(cv) exp_p = 1.0 - errmax = discr.norm(p - exp_p, np.inf) + errmax = inf_norm(p - exp_p) exp_ke = 0.5 * cv.mass ke = eos.kinetic_energy(cv) - kerr = discr.norm(ke - exp_ke, np.inf) + kerr = inf_norm(ke - exp_ke) te = eos.total_energy(cv, p) - terr = discr.norm(te - cv.energy, np.inf) + terr = inf_norm(te - cv.energy) logger.info(f"lump_soln = {cv}") logger.info(f"pressure = {p}") @@ -446,17 +458,20 @@ def test_idealsingle_vortex(ctx_factory): vortex = Vortex2D() cv = vortex(nodes) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + gamma = eos.gamma() p = eos.pressure(cv) exp_p = cv.mass ** gamma - errmax = discr.norm(p - exp_p, np.inf) + errmax = inf_norm(p - exp_p) exp_ke = 0.5 * np.dot(cv.momentum, cv.momentum) / cv.mass ke = eos.kinetic_energy(cv) - kerr = discr.norm(ke - exp_ke, np.inf) + kerr = inf_norm(ke - exp_ke) te = eos.total_energy(cv, p) - terr = discr.norm(te - cv.energy, np.inf) + terr = inf_norm(te - cv.energy) logger.info(f"vortex_soln = {cv}") logger.info(f"pressure = {p}") diff --git a/test/test_euler.py b/test/test_euler.py index 1f7b3daa6..821e9b43e 100644 --- a/test/test_euler.py +++ b/test/test_euler.py @@ -137,14 +137,17 @@ def test_uniform_rhs(actx_factory, nspecies, dim, order): f"rhoy_rhs = {rhoy_rhs}\n" ) - assert discr.norm(rho_resid, np.inf) < tolerance - assert discr.norm(rhoe_resid, np.inf) < tolerance + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + assert inf_norm(rho_resid) < tolerance + assert inf_norm(rhoe_resid) < tolerance for i in range(dim): - assert discr.norm(mom_resid[i], np.inf) < tolerance + assert inf_norm(mom_resid[i]) < tolerance for i in range(nspecies): - assert discr.norm(rhoy_resid[i], np.inf) < tolerance + assert inf_norm(rhoy_resid[i]) < tolerance - err_max = discr.norm(rho_resid, np.inf) + err_max = inf_norm(rho_resid) eoc_rec0.add_data_point(1.0 / nel_1d, err_max) # set a non-zero, but uniform velocity component @@ -165,15 +168,15 @@ def test_uniform_rhs(actx_factory, nspecies, dim, order): mom_resid = rhs_resid.momentum rhoy_resid = rhs_resid.species_mass - assert discr.norm(rho_resid, np.inf) < tolerance - assert discr.norm(rhoe_resid, np.inf) < tolerance + assert inf_norm(rho_resid) < tolerance + assert inf_norm(rhoe_resid) < tolerance for i in range(dim): - assert discr.norm(mom_resid[i], np.inf) < tolerance + assert inf_norm(mom_resid[i]) < tolerance for i in range(nspecies): - assert discr.norm(rhoy_resid[i], np.inf) < tolerance + assert inf_norm(rhoy_resid[i]) < tolerance - err_max = discr.norm(rho_resid, np.inf) + err_max = inf_norm(rho_resid) eoc_rec1.add_data_point(1.0 / nel_1d, err_max) logger.info( @@ -231,7 +234,7 @@ def test_vortex_rhs(actx_factory, order): discr, eos=IdealSingleGas(), boundaries=boundaries, cv=vortex_soln, time=0.0) - err_max = discr.norm(inviscid_rhs.join(), np.inf) + err_max = actx.to_numpy(discr.norm(inviscid_rhs.join(), np.inf)) eoc_rec.add_data_point(1.0 / nel_1d, err_max) logger.info( @@ -290,7 +293,8 @@ def test_lump_rhs(actx_factory, dim, order): ) expected_rhs = lump.exact_rhs(discr, cv=lump_soln, time=0) - err_max = discr.norm((inviscid_rhs-expected_rhs).join(), np.inf) + err_max = actx.to_numpy( + discr.norm((inviscid_rhs-expected_rhs).join(), np.inf)) if err_max > maxxerr: maxxerr = err_max @@ -365,7 +369,9 @@ def test_multilump_rhs(actx_factory, dim, order, v0): print(f"inviscid_rhs = {inviscid_rhs}") print(f"expected_rhs = {expected_rhs}") - err_max = discr.norm((inviscid_rhs-expected_rhs).join(), np.inf) + + err_max = actx.to_numpy( + discr.norm((inviscid_rhs-expected_rhs).join(), np.inf)) if err_max > maxxerr: maxxerr = err_max @@ -500,7 +506,7 @@ def rhs(t, q): maxerr = max(write_soln(cv, False)) else: expected_result = initializer(nodes, time=t) - maxerr = discr.norm((cv - expected_result).join(), np.inf) + maxerr = actx.to_numpy(discr.norm((cv - expected_result).join(), np.inf)) logger.info(f"Max Error: {maxerr}") if maxerr > exittol: diff --git a/test/test_filter.py b/test/test_filter.py index 78b7d5028..85814fb9c 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -192,7 +192,7 @@ def test_filter_function(actx_factory, dim, order, do_viz=False): filtered_soln = filter_modally(discr, "vol", cutoff, frfunc, uniform_soln) soln_resid = uniform_soln - filtered_soln - max_errors = [discr.norm(v, np.inf) for v in soln_resid] + max_errors = [actx.to_numpy(discr.norm(v, np.inf)) for v in soln_resid] tol = 1e-14 @@ -217,7 +217,7 @@ def polyfn(coeff): # , x_vec): filtered_field = filter_modally(discr, "vol", cutoff, frfunc, field) soln_resid = field - filtered_field - max_errors = [discr.norm(v, np.inf) for v in soln_resid] + max_errors = [actx.to_numpy(discr.norm(v, np.inf)) for v in soln_resid] logger.info(f"Field = {field}") logger.info(f"Filtered = {filtered_field}") logger.info(f"Max Errors (poly) = {max_errors}") @@ -253,6 +253,6 @@ def polyfn(coeff): # , x_vec): ] vis.write_vtk_file(f"filter_test_{field_order}.vtu", io_fields) field_resid = unfiltered_spectrum - filtered_spectrum - max_errors = [discr.norm(v, np.inf) for v in field_resid] + max_errors = [actx.to_numpy(discr.norm(v, np.inf)) for v in field_resid] # fields should be different, but not too different assert(tol > np.max(max_errors) > threshold) diff --git a/test/test_fluid.py b/test/test_fluid.py index 4f24e0581..9cac5d7aa 100644 --- a/test/test_fluid.py +++ b/test/test_fluid.py @@ -84,7 +84,7 @@ def test_velocity_gradient_sanity(actx_factory, dim, mass_exp, vel_fac): tol = 1e-11 exp_result = vel_fac * np.eye(dim) * ones - grad_v_err = [discr.norm(grad_v[i] - exp_result[i], np.inf) + grad_v_err = [actx.to_numpy(discr.norm(grad_v[i] - exp_result[i], np.inf)) for i in range(dim)] assert max(grad_v_err) < tol @@ -133,7 +133,8 @@ def exact_grad_row(xdata, gdim, dim): return exact_grad_row comp_err = make_obj_array([ - discr.norm(grad_v[i] - exact_grad_row(nodes[i], i, dim), np.inf) + actx.to_numpy( + discr.norm(grad_v[i] - exact_grad_row(nodes[i], i, dim), np.inf)) for i in range(dim)]) err_max = comp_err.max() eoc.add_data_point(h, err_max) @@ -188,9 +189,13 @@ def test_velocity_gradient_structure(actx_factory): assert grad_v.shape == (dim, dim) from meshmode.dof_array import DOFArray assert type(grad_v[0, 0]) == DOFArray - assert discr.norm(grad_v - exp_result, np.inf) < tol - assert discr.norm(grad_v.T - exp_trans, np.inf) < tol - assert discr.norm(np.trace(grad_v) - exp_trace, np.inf) < tol + + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + assert inf_norm(grad_v - exp_result) < tol + assert inf_norm(grad_v.T - exp_trans) < tol + assert inf_norm(np.trace(grad_v) - exp_trace) < tol @pytest.mark.parametrize("dim", [1, 2, 3]) @@ -237,10 +242,13 @@ def test_species_mass_gradient(actx_factory, dim): from meshmode.dof_array import DOFArray assert type(grad_y[0, 0]) == DOFArray + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + tol = 1e-11 for idim in range(dim): ispec = 2*idim exact_grad = np.array([(ispec*(idim*dim+1))*(iidim+1) for iidim in range(dim)]) - assert discr.norm(grad_y[ispec] - exact_grad, np.inf) < tol - assert discr.norm(grad_y[ispec+1] + exact_grad, np.inf) < tol + assert inf_norm(grad_y[ispec] - exact_grad) < tol + assert inf_norm(grad_y[ispec+1] + exact_grad) < tol diff --git a/test/test_init.py b/test/test_init.py index d2d85af99..c65c445db 100644 --- a/test/test_init.py +++ b/test/test_init.py @@ -84,7 +84,7 @@ def test_uniform_init(ctx_factory, dim, nspecies): def inf_norm(data): if len(data) > 0: - return discr.norm(data, np.inf) + return actx.to_numpy(discr.norm(data, np.inf)) else: return 0.0 @@ -140,7 +140,7 @@ def test_lump_init(ctx_factory): p = 0.4 * (cv.energy - 0.5 * np.dot(cv.momentum, cv.momentum) / cv.mass) exp_p = 1.0 - errmax = discr.norm(p - exp_p, np.inf) + errmax = actx.to_numpy(discr.norm(p - exp_p, np.inf)) logger.info(f"lump_soln = {cv}") logger.info(f"pressure = {p}") @@ -177,7 +177,7 @@ def test_vortex_init(ctx_factory): gamma = 1.4 p = 0.4 * (cv.energy - 0.5 * np.dot(cv.momentum, cv.momentum) / cv.mass) exp_p = cv.mass ** gamma - errmax = discr.norm(p - exp_p, np.inf) + errmax = actx.to_numpy(discr.norm(p - exp_p, np.inf)) logger.info(f"vortex_soln = {cv}") logger.info(f"pressure = {p}") @@ -220,7 +220,8 @@ def test_shock_init(ctx_factory): eos = IdealSingleGas() p = eos.pressure(initsoln) - assert discr.norm(actx.np.where(nodes_x < 0.5, p-xpl, p-xpr), np.inf) < tol + assert actx.to_numpy( + discr.norm(actx.np.where(nodes_x < 0.5, p-xpl, p-xpr), np.inf)) < tol @pytest.mark.parametrize("dim", [1, 2, 3]) @@ -254,15 +255,18 @@ def test_uniform(ctx_factory, dim): initsoln = initr(time=0.0, x_vec=nodes) tol = 1e-15 - assert discr.norm(initsoln.mass - 1.0, np.inf) < tol - assert discr.norm(initsoln.energy - 2.5, np.inf) < tol + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + assert inf_norm(initsoln.mass - 1.0) < tol + assert inf_norm(initsoln.energy - 2.5) < tol print(f"Uniform Soln:{initsoln}") eos = IdealSingleGas() p = eos.pressure(initsoln) print(f"Press:{p}") - assert discr.norm(p - 1.0, np.inf) < tol + assert inf_norm(p - 1.0) < tol @pytest.mark.parametrize("dim", [1, 2, 3]) @@ -301,30 +305,33 @@ def test_pulse(ctx_factory, dim): pulse = make_pulse(amp=amp, r0=r0, w=w, r=nodes) print(f"Pulse = {pulse}") + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + # does it return the expected exponential? pulse_check = actx.np.exp(-.5 * r2) print(f"exact: {pulse_check}") pulse_resid = pulse - pulse_check print(f"pulse residual: {pulse_resid}") - assert(discr.norm(pulse_resid, np.inf) < tol) + assert(inf_norm(pulse_resid) < tol) # proper scaling with amplitude? amp = 2.0 pulse = 0 pulse = make_pulse(amp=amp, r0=r0, w=w, r=nodes) pulse_resid = pulse - (pulse_check + pulse_check) - assert(discr.norm(pulse_resid, np.inf) < tol) + assert(inf_norm(pulse_resid) < tol) # proper scaling with r? amp = 1.0 rcheck = np.sqrt(2.0) * nodes pulse = make_pulse(amp=amp, r0=r0, w=w, r=rcheck) - assert(discr.norm(pulse - (pulse_check * pulse_check), np.inf) < tol) + assert(inf_norm(pulse - (pulse_check * pulse_check)) < tol) # proper scaling with w? w = w / np.sqrt(2.0) pulse = make_pulse(amp=amp, r0=r0, w=w, r=nodes) - assert(discr.norm(pulse - (pulse_check * pulse_check), np.inf) < tol) + assert(inf_norm(pulse - (pulse_check * pulse_check)) < tol) @pytest.mark.parametrize("dim", [1, 2, 3]) @@ -370,12 +377,15 @@ def test_multilump(ctx_factory, dim): numcvspec = len(cv.species_mass) print(f"get_num_species = {numcvspec}") + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + assert numcvspec == nspecies - assert discr.norm(cv.mass - rho0) == 0.0 + assert inf_norm(cv.mass - rho0) == 0.0 p = 0.4 * (cv.energy - 0.5 * np.dot(cv.momentum, cv.momentum) / cv.mass) exp_p = 1.0 - errmax = discr.norm(p - exp_p, np.inf) + errmax = inf_norm(p - exp_p) species_mass = cv.species_mass spec_r = make_obj_array([nodes - centers[i] for i in range(nspecies)]) @@ -389,7 +399,7 @@ def test_multilump(ctx_factory, dim): print(f"exp_mass = {exp_mass}") print(f"mass_resid = {mass_resid}") - assert discr.norm(mass_resid, np.inf) == 0.0 + assert inf_norm(mass_resid) == 0.0 logger.info(f"lump_soln = {cv}") logger.info(f"pressure = {p}") diff --git a/test/test_inviscid.py b/test/test_inviscid.py index 37f0cea94..c97b37501 100644 --- a/test/test_inviscid.py +++ b/test/test_inviscid.py @@ -175,16 +175,20 @@ def test_inviscid_flux_components(actx_factory, dim): cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) p = eos.pressure(cv) flux = inviscid_flux(discr, eos, cv) - assert discr.norm(p - p_exact, np.inf) < tolerance + + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + assert inf_norm(p - p_exact) < tolerance logger.info(f"{dim}d flux = {flux}") # for velocity zero, these components should be == zero - assert discr.norm(flux.mass, 2) == 0.0 - assert discr.norm(flux.energy, 2) == 0.0 + assert inf_norm(flux.mass) == 0.0 + assert inf_norm(flux.energy) == 0.0 # The momentum diagonal should be p # Off-diagonal should be identically 0 - assert discr.norm(flux.momentum - p0*np.identity(dim), np.inf) < tolerance + assert inf_norm(flux.momentum - p0*np.identity(dim)) < tolerance @pytest.mark.parametrize(("dim", "livedim"), [ @@ -230,19 +234,23 @@ def test_inviscid_mom_flux_components(actx_factory, dim, livedim): ) cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) p = eos.pressure(cv) - assert discr.norm(p - p_exact, np.inf) < tolerance + + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + assert inf_norm(p - p_exact) < tolerance flux = inviscid_flux(discr, eos, cv) logger.info(f"{dim}d flux = {flux}") vel_exact = mom / mass # first two components should be nonzero in livedim only - assert discr.norm(flux.mass - mom, np.inf) == 0 + assert inf_norm(flux.mass - mom) == 0 eflux_exact = (energy + p_exact)*vel_exact - assert discr.norm(flux.energy - eflux_exact, np.inf) == 0 + assert inf_norm(flux.energy - eflux_exact) == 0 logger.info("Testing momentum") xpmomflux = mass*np.outer(vel_exact, vel_exact) + p_exact*np.identity(dim) - assert discr.norm(flux.momentum - xpmomflux, np.inf) < tolerance + assert inf_norm(flux.momentum - xpmomflux) < tolerance @pytest.mark.parametrize("nspecies", [0, 10]) @@ -302,7 +310,7 @@ def test_facial_flux(actx_factory, nspecies, order, dim): def inf_norm(data): if len(data) > 0: - return discr.norm(data, np.inf, dd="all_faces") + return actx.to_numpy(discr.norm(data, np.inf, dd="all_faces")) else: return 0.0 diff --git a/test/test_lazy.py b/test/test_lazy.py index e4af85344..75e645e0f 100644 --- a/test/test_lazy.py +++ b/test/test_lazy.py @@ -79,7 +79,9 @@ def componentwise_norm(a): from mirgecom.fluid import ConservedVars if isinstance(a, ConservedVars): return componentwise_norm(a.join()) - return obj_array_vectorize(lambda b: discr.norm(b, np.inf), a) + from arraycontext import get_container_context_recursively + actx = get_container_context_recursively(a) + return obj_array_vectorize(lambda b: actx.to_numpy(discr.norm(b, np.inf)), a) lhs = componentwise_norm(x - y) rhs = np.maximum( diff --git a/test/test_operators.py b/test/test_operators.py index 049a67b31..bff1f2146 100644 --- a/test/test_operators.py +++ b/test/test_operators.py @@ -217,10 +217,13 @@ def sym_eval(expr, x_vec): test_data = test_func(nodes) exact_grad = grad_test_func(nodes) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + if isinstance(test_data, ConservedVars): - err_scale = discr.norm(exact_grad.join(), np.inf) + err_scale = inf_norm(exact_grad.join()) else: - err_scale = discr.norm(exact_grad, np.inf) + err_scale = inf_norm(exact_grad) if err_scale <= 1e-16: err_scale = 1 @@ -238,10 +241,10 @@ def sym_eval(expr, x_vec): dim=dim, q=grad_operator(discr, test_data.join(), test_data_flux_bnd.join()) ) - grad_err = discr.norm((test_grad - exact_grad).join(), np.inf)/err_scale + grad_err = inf_norm((test_grad - exact_grad).join())/err_scale else: test_grad = grad_operator(discr, test_data, test_data_flux_bnd) - grad_err = discr.norm(test_grad - exact_grad, np.inf)/err_scale + grad_err = inf_norm(test_grad - exact_grad)/err_scale print(f"{test_grad=}") eoc.add_data_point(h_max, grad_err) diff --git a/test/test_restart.py b/test/test_restart.py index 287598e01..2b6b6dcdb 100644 --- a/test/test_restart.py +++ b/test/test_restart.py @@ -76,4 +76,4 @@ def test_restart_cv(actx_factory, nspecies): restart_data = read_restart_data(actx, rst_filename) resid = test_state - restart_data["state"] - assert discr.norm(resid.join(), np.inf) == 0 + assert actx.to_numpy(discr.norm(resid.join(), np.inf)) == 0 diff --git a/test/test_simutil.py b/test/test_simutil.py index 9f78062db..a590abcd9 100644 --- a/test/test_simutil.py +++ b/test/test_simutil.py @@ -134,7 +134,7 @@ def test_analytic_comparison(actx_factory): cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) resid = vortex_soln - cv - expected_errors = [discr.norm(v, np.inf) for v in resid.join()] + expected_errors = [actx.to_numpy(discr.norm(v, np.inf)) for v in resid.join()] errors = compare_fluid_solutions(discr, cv, cv) assert max(errors) == 0 diff --git a/test/test_symbolic.py b/test/test_symbolic.py index c2337f1e6..489476014 100644 --- a/test/test_symbolic.py +++ b/test/test_symbolic.py @@ -164,7 +164,7 @@ def test_symbolic_evaluation(actx_factory): expected_f = np.exp(-t) * actx.np.cos(nodes[0]) * actx.np.sin(nodes[1]) - assert discr.norm(f - expected_f)/discr.norm(expected_f) < 1e-12 + assert actx.to_numpy(discr.norm(f - expected_f)/discr.norm(expected_f)) < 1e-12 if __name__ == "__main__": diff --git a/test/test_viscous.py b/test/test_viscous.py index 3fffb39e0..e132318ee 100644 --- a/test/test_viscous.py +++ b/test/test_viscous.py @@ -105,7 +105,7 @@ def test_viscous_stress_tensor(actx_factory, transport_model): tau = viscous_stress_tensor(discr, eos, cv, grad_cv) # The errors come from grad_v - assert discr.norm(tau - exp_tau, np.inf) < 1e-12 + assert actx.to_numpy(discr.norm(tau - exp_tau, np.inf)) < 1e-12 # Box grid generator widget lifted from @majosm and slightly bent @@ -189,6 +189,9 @@ def cv_flux_boundary(btag): discr = EagerDGDiscretization(actx, mesh, order=order) nodes = thaw(actx, discr.nodes()) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + # compute max element size from grudge.dt_utils import h_max_from_volume h_max = h_max_from_volume(discr) @@ -208,10 +211,10 @@ def cv_flux_boundary(btag): xp_tau = mu * (xp_grad_v + xp_grad_v.transpose()) # sanity check the gradient: - relerr_scale_e = 1.0 / discr.norm(xp_grad_cv.energy, np.inf) - relerr_scale_p = 1.0 / discr.norm(xp_grad_cv.momentum, np.inf) - graderr_e = discr.norm((grad_cv.energy - xp_grad_cv.energy), np.inf) - graderr_p = discr.norm((grad_cv.momentum - xp_grad_cv.momentum), np.inf) + relerr_scale_e = 1.0 / inf_norm(xp_grad_cv.energy) + relerr_scale_p = 1.0 / inf_norm(xp_grad_cv.momentum) + graderr_e = inf_norm(grad_cv.energy - xp_grad_cv.energy) + graderr_p = inf_norm(grad_cv.momentum - xp_grad_cv.momentum) graderr_e *= relerr_scale_e graderr_p *= relerr_scale_p assert graderr_e < 5e-7 @@ -231,14 +234,14 @@ def cv_flux_boundary(btag): grad_t = op.local_grad(discr, temperature) # sanity check - assert discr.norm(grad_p - xp_grad_p, np.inf)*dpscal < 5e-9 - assert discr.norm(grad_t - xp_grad_t, np.inf)*tscal < 5e-9 + assert inf_norm(grad_p - xp_grad_p)*dpscal < 5e-9 + assert inf_norm(grad_t - xp_grad_t)*tscal < 5e-9 # verify heat flux from mirgecom.viscous import conductive_heat_flux heat_flux = conductive_heat_flux(discr, eos, cv, grad_t) xp_heat_flux = -kappa*xp_grad_t - assert discr.norm(heat_flux - xp_heat_flux, np.inf) < 2e-8 + assert inf_norm(heat_flux - xp_heat_flux) < 2e-8 # verify diffusive mass flux is zilch (no scalar components) from mirgecom.viscous import diffusive_flux @@ -251,15 +254,15 @@ def cv_flux_boundary(btag): vflux = viscous_flux(discr, eos, cv, grad_cv, grad_t) efluxerr = ( - discr.norm(vflux.energy - xp_e_flux, np.inf) - / discr.norm(xp_e_flux, np.inf) + inf_norm(vflux.energy - xp_e_flux) + / inf_norm(xp_e_flux) ) momfluxerr = ( - discr.norm(vflux.momentum - xp_mom_flux, np.inf) - / discr.norm(xp_mom_flux, np.inf) + inf_norm(vflux.momentum - xp_mom_flux) + / inf_norm(xp_mom_flux) ) - assert discr.norm(vflux.mass, np.inf) == 0 + assert inf_norm(vflux.mass) == 0 e_eoc_rec.add_data_point(h_max, efluxerr) p_eoc_rec.add_data_point(h_max, momfluxerr) @@ -333,15 +336,18 @@ def test_species_diffusive_flux(actx_factory): from mirgecom.viscous import diffusive_flux j = diffusive_flux(discr, eos, cv, grad_cv) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + tol = 1e-10 for idim in range(dim): ispec = 2*idim exact_dy = np.array([((ispec+1)*(idim*dim+1))*(iidim+1) for iidim in range(dim)]) exact_j = -massval * d_alpha[ispec] * exact_dy - assert discr.norm(j[ispec] - exact_j, np.inf) < tol + assert inf_norm(j[ispec] - exact_j) < tol exact_j = massval * d_alpha[ispec+1] * exact_dy - assert discr.norm(j[ispec+1] - exact_j, np.inf) < tol + assert inf_norm(j[ispec+1] - exact_j) < tol def test_diffusive_heat_flux(actx_factory): @@ -403,15 +409,18 @@ def test_diffusive_heat_flux(actx_factory): from mirgecom.viscous import diffusive_flux j = diffusive_flux(discr, eos, cv, grad_cv) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + tol = 1e-10 for idim in range(dim): ispec = 2*idim exact_dy = np.array([((ispec+1)*(idim*dim+1))*(iidim+1) for iidim in range(dim)]) exact_j = -massval * d_alpha[ispec] * exact_dy - assert discr.norm(j[ispec] - exact_j, np.inf) < tol + assert inf_norm(j[ispec] - exact_j) < tol exact_j = massval * d_alpha[ispec+1] * exact_dy - assert discr.norm(j[ispec+1] - exact_j, np.inf) < tol + assert inf_norm(j[ispec+1] - exact_j) < tol @pytest.mark.parametrize("array_valued", [False, True]) @@ -462,7 +471,7 @@ def test_local_max_species_diffusivity(actx_factory, dim, array_valued): expected *= f calculated = get_local_max_species_diffusivity(actx, discr, d_alpha) - assert discr.norm(calculated-expected, np.inf) == 0 + assert actx.to_numpy(discr.norm(calculated-expected, np.inf)) == 0 @pytest.mark.parametrize("dim", [1, 2, 3]) @@ -519,4 +528,4 @@ def test_viscous_timestep(actx_factory, dim, mu, vel): dt_expected = chlen / (speed_total + (mu / chlen)) error = (dt_expected - dt_field) / dt_expected - assert discr.norm(error, np.inf) == 0 + assert actx.to_numpy(discr.norm(error, np.inf)) == 0 diff --git a/test/test_wave.py b/test/test_wave.py index d33c98259..5e4cf8d3b 100644 --- a/test/test_wave.py +++ b/test/test_wave.py @@ -184,7 +184,7 @@ def sym_eval(expr, t): expected_rhs = sym_eval(sym_rhs, t_check) - rel_linf_err = ( + rel_linf_err = actx.to_numpy( discr.norm(rhs - expected_rhs, np.inf) / discr.norm(expected_rhs, np.inf)) eoc_rec.add_data_point(1./n, rel_linf_err) @@ -269,8 +269,11 @@ def get_rhs(t, w): ("v_expected", expected_fields[1:]), ]) - err = discr.norm(fields-expected_fields, np.inf) - max_err = discr.norm(expected_fields, np.inf) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + err = inf_norm(fields-expected_fields) + max_err = inf_norm(expected_fields) assert err < max_err From 599d7e62cc5ebb5457b6732ffff87e2685125e9d Mon Sep 17 00:00:00 2001 From: Matthew Smith Date: Mon, 11 Oct 2021 12:08:04 -0500 Subject: [PATCH 324/385] temporarily change grudge branch --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4dc2b033d..ddfe23291 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ pyyaml --editable git+https://github.com/inducer/modepy.git#egg=modepy --editable git+https://github.com/inducer/arraycontext.git#egg=arraycontext --editable git+https://github.com/inducer/meshmode.git#egg=meshmode ---editable git+https://github.com/inducer/grudge.git#egg=grudge +--editable git+https://github.com/majosm/grudge.git@nodal-reduction-device-scalar#egg=grudge --editable git+https://github.com/inducer/pytato.git#egg=pytato --editable git+https://github.com/ecisneros8/pyrometheus.git#egg=pyrometheus --editable git+https://github.com/illinois-ceesd/logpyle.git#egg=logpyle From dfb511da2a8a6d4302f5c034bc6027600ece9708 Mon Sep 17 00:00:00 2001 From: Matthew Smith Date: Wed, 20 Oct 2021 11:31:41 -0500 Subject: [PATCH 325/385] account for h_min/max_from_volume returning device scalars --- test/test_operators.py | 2 +- test/test_viscous.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_operators.py b/test/test_operators.py index bff1f2146..d60a8ec80 100644 --- a/test/test_operators.py +++ b/test/test_operators.py @@ -247,7 +247,7 @@ def inf_norm(x): grad_err = inf_norm(test_grad - exact_grad)/err_scale print(f"{test_grad=}") - eoc.add_data_point(h_max, grad_err) + eoc.add_data_point(actx.to_numpy(h_max), grad_err) assert ( eoc.order_estimate() >= order - 0.5 diff --git a/test/test_viscous.py b/test/test_viscous.py index e132318ee..d30e37cdf 100644 --- a/test/test_viscous.py +++ b/test/test_viscous.py @@ -263,8 +263,8 @@ def inf_norm(x): ) assert inf_norm(vflux.mass) == 0 - e_eoc_rec.add_data_point(h_max, efluxerr) - p_eoc_rec.add_data_point(h_max, momfluxerr) + e_eoc_rec.add_data_point(actx.to_numpy(h_max), efluxerr) + p_eoc_rec.add_data_point(actx.to_numpy(h_max), momfluxerr) assert ( e_eoc_rec.order_estimate() >= order - 0.5 From 47f144782258eae2b1fb39520e96f414ae176ff4 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Wed, 20 Oct 2021 19:23:39 -0500 Subject: [PATCH 326/385] use arraycontext.outer in flux functions (#517) --- mirgecom/flux.py | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/mirgecom/flux.py b/mirgecom/flux.py index e76ddc314..92d5ff838 100644 --- a/mirgecom/flux.py +++ b/mirgecom/flux.py @@ -33,11 +33,6 @@ THE SOFTWARE. """ import numpy as np # noqa -from meshmode.dof_array import DOFArray -from mirgecom.fluid import ( - ConservedVars, - make_conserved -) def gradient_flux_central(u_tpair, normal): @@ -74,22 +69,8 @@ def gradient_flux_central(u_tpair, normal): object array of :class:`~meshmode.dof_array.DOFArray` with the flux for each scalar component. """ - tp_avg = u_tpair.avg - tp_join = tp_avg - - # FIXME: There's a better way in-the-works through an improved "outer". - # Update when https://github.com/inducer/arraycontext/pull/46 lands. - if isinstance(tp_avg, DOFArray): - return tp_avg*normal - elif isinstance(tp_avg, ConservedVars): - tp_join = tp_avg.join() - - result = np.outer(tp_join, normal) - - if isinstance(tp_avg, ConservedVars): - return make_conserved(tp_avg.dim, q=result) - else: - return result + from arraycontext import outer + return outer(u_tpair.avg, normal) def divergence_flux_central(trace_pair, normal): @@ -228,6 +209,5 @@ def flux_lfr(cv_tpair, f_tpair, normal, lam): object array of :class:`~meshmode.dof_array.DOFArray` with the Lax-Friedrichs/Rusanov flux. """ - return make_conserved( - dim=len(normal), - q=f_tpair.avg.join() - lam*np.outer(cv_tpair.diff.join(), normal)/2) + from arraycontext import outer + return f_tpair.avg - lam*outer(cv_tpair.diff, normal)/2 From 50cd5dd5013e6b76bf77b08948c501fc9fdce63c Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Sun, 24 Oct 2021 15:27:48 -0500 Subject: [PATCH 327/385] Update ci.yaml (#529) --- .github/workflows/ci.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ecba113d5..3d0ff1d31 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -79,7 +79,8 @@ jobs: MINIFORGE_INSTALL_DIR=.miniforge3 . "$MINIFORGE_INSTALL_DIR/bin/activate" testing cd test - python -m pytest --cov=mirgecom --durations=0 --tb=native --junitxml=pytest.xml --doctest-modules -rxsw . ../doc/*.rst ../doc/*/*.rst + python -m pip install pytest-xdist + python -m pytest -n auto --cov=mirgecom --durations=0 --tb=native --junitxml=pytest.xml --doctest-modules -rxsw . ../doc/*.rst ../doc/*/*.rst examples: name: Examples From 9cc8d602b51c50234ef0a4fe9b8981965801b473 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Mon, 25 Oct 2021 18:20:52 -0500 Subject: [PATCH 328/385] Use new thermochem wrapper class (#512) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matt Smith Co-authored-by: Andreas Klöckner --- doc/support/thermochem.rst | 1 + examples/autoignition-mpi.py | 50 +++++++++++--- examples/mixture-mpi.py | 4 +- mirgecom/eos.py | 3 +- mirgecom/thermochemistry.py | 126 +++++++++++++++++++++++++++++++++++ test/test_eos.py | 12 ++-- 6 files changed, 178 insertions(+), 18 deletions(-) create mode 100644 mirgecom/thermochemistry.py diff --git a/doc/support/thermochem.rst b/doc/support/thermochem.rst index 864019825..21a0ae4e9 100644 --- a/doc/support/thermochem.rst +++ b/doc/support/thermochem.rst @@ -2,3 +2,4 @@ Thermochemistry mechanism management ==================================== .. automodule:: mirgecom.mechanisms +.. automodule:: mirgecom.thermochemistry diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 5ec03576f..a60dda8e1 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -67,7 +67,6 @@ ) import cantera -import pyrometheus as pyro logger = logging.getLogger(__name__) @@ -249,9 +248,9 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, # Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and # generates a set of methods to calculate chemothermomechanical properties and # states for this particular mechanism. - pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np) - eos = PyrometheusMixture(pyrometheus_mechanism, - temperature_guess=init_temperature) + from mirgecom.thermochemistry import make_pyrometheus_mechanism_class + pyro_mechanism = make_pyrometheus_mechanism_class(cantera_soln)(actx.np) + eos = PyrometheusMixture(pyro_mechanism, temperature_guess=init_temperature) # }}} @@ -321,6 +320,17 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, f" {eq_pressure=}, {eq_temperature=}," f" {eq_density=}, {eq_mass_fractions=}") + from pytools.obj_array import make_obj_array + + def get_temperature_mass_energy(state, temperature): + y = state.species_mass_fractions + e = eos.internal_energy(state) / state.mass + return make_obj_array( + [pyro_mechanism.get_temperature(e, temperature, y)] + ) + + compute_temperature = actx.compile(get_temperature_mass_energy) + def my_write_status(dt, cfl): status_msg = f"------ {dt=}" if constant_cfl else f"----- {cfl=}" if rank == 0: @@ -359,18 +369,40 @@ def my_write_restart(step, t, state): from mirgecom.restart import write_restart_file write_restart_file(actx, rst_data, rst_fname, comm) - def my_health_check(dv): + def my_health_check(cv, dv): health_error = False + pressure = dv.pressure + temperature = dv.temperature from mirgecom.simutil import check_naninf_local, check_range_local - if check_naninf_local(discr, "vol", dv.pressure) \ - or check_range_local(discr, "vol", dv.pressure, 1e5, 2.4e5): + if check_naninf_local(discr, "vol", pressure) \ + or check_range_local(discr, "vol", pressure, 1e5, 2.4e5): health_error = True logger.info(f"{rank=}: Invalid pressure data found.") - if check_range_local(discr, "vol", dv.temperature, 1.498e3, 1.52e3): + if check_range_local(discr, "vol", temperature, 1.498e3, 1.52e3): health_error = True logger.info(f"{rank=}: Invalid temperature data found.") + # This check is the temperature convergence check + # The current *temperature* is what Pyrometheus gets + # after a fixed number of Newton iterations, *n_iter*. + # Calling `compute_temperature` here with *temperature* + # input as the guess returns the calculated gas temperature after + # yet another *n_iter*. + # The difference between those two temperatures is the + # temperature residual, which can be used as an indicator of + # convergence in Pyrometheus `get_temperature`. + # Note: The local max jig below works around a very long compile + # in lazy mode. + from grudge.op import nodal_max_loc + check_temp, = compute_temperature(cv, temperature) + temp_resid = actx.np.abs(check_temp - temperature) + temp_resid = nodal_max_loc(discr, "vol", temp_resid) + + if temp_resid > 1e-12: + health_error = True + logger.info(f"{rank=}: Temperature is not converged {temp_resid=}.") + return health_error def my_get_timestep(t, dt, state): @@ -405,7 +437,7 @@ def my_pre_step(step, t, dt, state): if do_health: dv = eos.dependent_vars(state) - health_errors = global_reduce(my_health_check(dv), op="lor") + health_errors = global_reduce(my_health_check(state, dv), op="lor") if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") diff --git a/examples/mixture-mpi.py b/examples/mixture-mpi.py index 01fb26291..a0905c017 100644 --- a/examples/mixture-mpi.py +++ b/examples/mixture-mpi.py @@ -55,7 +55,6 @@ from mirgecom.eos import PyrometheusMixture import cantera -import pyrometheus as pyro from logpyle import IntervalTimer, set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging @@ -180,7 +179,8 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, from mirgecom.mechanisms import get_mechanism_cti mech_cti = get_mechanism_cti("uiuc") sol = cantera.Solution(phase_id="gas", source=mech_cti) - pyrometheus_mechanism = pyro.get_thermochem_class(sol)(actx.np) + from mirgecom.thermochemistry import make_pyrometheus_mechanism_class + pyrometheus_mechanism = make_pyrometheus_mechanism_class(sol)(actx.np) nspecies = pyrometheus_mechanism.num_species eos = PyrometheusMixture(pyrometheus_mechanism) diff --git a/mirgecom/eos.py b/mirgecom/eos.py index 2cdef4057..106bf579b 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -749,8 +749,7 @@ def temperature(self, cv: ConservedVars): def get_temp(): y = cv.species_mass_fractions e = self.internal_energy(cv) / cv.mass - return self._pyrometheus_mech.get_temperature(e, self._tguess, - y, True) + return self._pyrometheus_mech.get_temperature(e, self._tguess, y) return get_temp() def total_energy(self, cv, pressure): diff --git a/mirgecom/thermochemistry.py b/mirgecom/thermochemistry.py new file mode 100644 index 000000000..d4863ac88 --- /dev/null +++ b/mirgecom/thermochemistry.py @@ -0,0 +1,126 @@ +r""":mod:`mirgecom.thermochemistry` provides a wrapper class for :mod:`pyrometheus`.. + +.. autofunction:: make_pyrometheus_mechanism_class +""" + +__copyright__ = """ +Copyright (C) 2021 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + + +def _pyro_thermochem_wrapper_class(cantera_soln, temperature_niter=5): + """Return a MIRGE-compatible wrapper for a :mod:`pyrometheus` mechanism class. + + Dynamically creates a class that inherits from a + :class:`pyrometheus.Thermochemistry` class and overrides a couple of the methods + to adapt it to :mod:`mirgecom`'s needs. + + - get_concentrations: overrides :class:`pyrometheus.Thermochemistry` version + of the same function, pinning any negative concentrations due to slightly + negative massfractions (which are OK) back to 0. + - get_temperature: MIRGE-specific interface to use a hard-coded Newton solver + to find a temperature from an input state. + + Parameters + ---------- + cantera_soln: Cantera solution + Cantera solution from which to create the thermochemical mechanism + temperature_niter: integer + Number of Newton iterations in `get_temperature` (default=5) + """ + import pyrometheus as pyro + pyro_class = pyro.get_thermochem_class(cantera_soln) + + class PyroWrapper(pyro_class): + + # This bit disallows negative concentrations and instead + # pins them to 0. mass_fractions can sometimes be slightly + # negative and that's ok. + def get_concentrations(self, rho, mass_fractions): + concs = self.iwts * rho * mass_fractions + # ensure non-negative concentrations + zero = self._pyro_zeros_like(concs[0]) + for i in range(self.num_species): + concs[i] = self.usr_np.maximum(concs[i], zero) + return concs + + # This is the temperature update for *get_temperature* + def _get_temperature_update_energy(self, e_in, t_in, y): + pv_func = self.get_mixture_specific_heat_cv_mass + he_func = self.get_mixture_internal_energy_mass + return (e_in - he_func(t_in, y)) / pv_func(t_in, y) + + # This hard-codes the number of Newton iterations because the convergence + # check is not compatible with lazy evaluation. Instead, we plan to check + # the temperature residual at simulation health checking time. + # FIXME: Occasional convergence check is other-than-ideal; revisit asap. + # - could adapt dt or num_iter on temperature convergence? + # - can pass-in num_iter? + def get_temperature(self, energy, temperature_guess, species_mass_fractions): + """Compute the temperature of the mixture from thermal energy. + + Parameters + ---------- + energy: :class:`~meshmode.dof_array.DOFArray` + The internal (thermal) energy of the mixture. + temperature_guess: :class:`~meshmode.dof_array.DOFArray` + An initial starting temperature for the Newton iterations. + species_mass_fractions: numpy.ndarray + An object array of :class:`~meshmode.dof_array.DOFArray` with the + mass fractions of the mixture species. + + Returns + ------- + :class:`~meshmode.dof_array.DOFArray` + The mixture temperature after a fixed number of Newton iterations. + """ + num_iter = temperature_niter + t_i = temperature_guess + for _ in range(num_iter): + t_i = t_i + self._get_temperature_update_energy( + energy, t_i, species_mass_fractions + ) + return t_i + + return PyroWrapper + + +def make_pyrometheus_mechanism_class(cantera_soln): + """Create a :mod:`pyrometheus` thermochemical (or equivalent) mechanism object. + + This routine creates and returns an instance of a :mod:`pyrometheus` + thermochemical mechanism for use in a MIRGE-Com fluid EOS. + + Parameters + ---------- + actx: :class:`arraycontext.ArrayContext` + Array context from which to get the numpy-like namespace for + :mod:`pyrometheus` + cantera_soln: + Cantera Solution for the thermochemical mechanism to be used + + Returns + ------- + :mod:`pyrometheus` ThermoChem class + """ + return _pyro_thermochem_wrapper_class(cantera_soln) diff --git a/test/test_eos.py b/test/test_eos.py index 7c90c87c9..124b5d6ff 100644 --- a/test/test_eos.py +++ b/test/test_eos.py @@ -90,7 +90,8 @@ def test_pyrometheus_mechanisms(ctx_factory, mechname, rate_tol, y0): # Pyrometheus initialization mech_cti = get_mechanism_cti(mechname) sol = cantera.Solution(phase_id="gas", source=mech_cti) - prometheus_mechanism = pyro.get_thermochem_class(sol)(actx.np) + from mirgecom.thermochemistry import make_pyrometheus_mechanism_class + prometheus_mechanism = make_pyrometheus_mechanism_class(sol)(actx.np) nspecies = prometheus_mechanism.num_species print(f"PyrometheusMixture::NumSpecies = {nspecies}") @@ -127,7 +128,7 @@ def test_pyrometheus_mechanisms(ctx_factory, mechname, rate_tol, y0): prom_rho = prometheus_mechanism.get_density(pin, tin, yin) prom_e = prometheus_mechanism.get_mixture_internal_energy_mass(tin, yin) - prom_t = prometheus_mechanism.get_temperature(prom_e, tin, yin, True) + prom_t = prometheus_mechanism.get_temperature(prom_e, tin, yin) prom_p = prometheus_mechanism.get_pressure(prom_rho, tin, yin) prom_c = prometheus_mechanism.get_concentrations(prom_rho, yin) prom_k = prometheus_mechanism.get_fwd_rate_coefficients(prom_t, prom_c) @@ -195,7 +196,8 @@ def test_pyrometheus_eos(ctx_factory, mechname, dim, y0, vel): # Pyrometheus initialization mech_cti = get_mechanism_cti(mechname) sol = cantera.Solution(phase_id="gas", source=mech_cti) - prometheus_mechanism = pyro.get_thermochem_class(sol)(actx.np) + from mirgecom.thermochemistry import make_pyrometheus_mechanism_class + prometheus_mechanism = make_pyrometheus_mechanism_class(sol)(actx.np) nspecies = prometheus_mechanism.num_species print(f"PrometheusMixture::Mechanism = {mechname}") @@ -209,7 +211,7 @@ def test_pyrometheus_eos(ctx_factory, mechname, dim, y0, vel): y0s[0] = 1.0 - np.sum(y0s[1:]) velocity = vel * np.ones(shape=(dim,)) - for fac in range(1, 11): + for fac in range(1, 7): tempin = fac * temp0 pressin = fac * press0 @@ -223,7 +225,7 @@ def test_pyrometheus_eos(ctx_factory, mechname, dim, y0, vel): pyro_rho = prometheus_mechanism.get_density(pin, tin, yin) pyro_e = prometheus_mechanism.get_mixture_internal_energy_mass(tin, yin) - pyro_t = prometheus_mechanism.get_temperature(pyro_e, tguess, yin, True) + pyro_t = prometheus_mechanism.get_temperature(pyro_e, tguess, yin) pyro_p = prometheus_mechanism.get_pressure(pyro_rho, pyro_t, yin) print(f"prom(rho, y, p, t, e) = ({pyro_rho}, {y0s}, " From 980f6dddf598ae58393a261f7ba1e308f3efcf38 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 26 Oct 2021 07:13:30 -0500 Subject: [PATCH 329/385] Add backwards-compatible thermochem interface. --- mirgecom/thermochemistry.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/mirgecom/thermochemistry.py b/mirgecom/thermochemistry.py index acff97001..81bbb9438 100644 --- a/mirgecom/thermochemistry.py +++ b/mirgecom/thermochemistry.py @@ -1,6 +1,7 @@ r""":mod:`mirgecom.thermochemistry` provides a wrapper class for :mod:`pyrometheus`.. .. autofunction:: make_pyrometheus_mechanism_class +.. autofunction:: make_pyrometheus_mechanism """ __copyright__ = """ @@ -106,7 +107,7 @@ def get_temperature(self, energy, temperature_guess, species_mass_fractions): return PyroWrapper -def make_pyrometheus_mechanism_class(cantera_soln): +def make_pyrometheus_mechanism_class(cantera_soln, temperature_niter=5): """Create a :mod:`pyrometheus` thermochemical (or equivalent) mechanism class. This routine creates and returns an instance of a :mod:`pyrometheus` @@ -124,4 +125,27 @@ def make_pyrometheus_mechanism_class(cantera_soln): ------- :mod:`pyrometheus` ThermoChem class """ - return _pyro_thermochem_wrapper_class(cantera_soln) + return _pyro_thermochem_wrapper_class(cantera_soln, temperature_niter) + + +def make_pyrometheus_mechanism(actx, cantera_soln): + """Create a :mod:`pyrometheus` thermochemical (or equivalent) mechanism. + + This routine creates and returns an instance of a :mod:`pyrometheus` + thermochemical mechanism for use in a MIRGE-Com fluid EOS. + + Parameters + ---------- + actx: :class:`arraycontext.ArrayContext` + Array context from which to get the numpy-like namespace for + :mod:`pyrometheus` + cantera_soln: + Cantera Solution for the thermochemical mechanism to be used + + Returns + ------- + :mod:`pyrometheus` ThermoChem class + """ + from warnings import warn + warn("make_pyrometheus_mechanism is deprecated and will disappear in Q1/2022") + return _pyro_thermochem_wrapper_class(cantera_soln)(actx.np) From 598519ece0150ec9e50631e29d0f58e9e5b33383 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 26 Oct 2021 07:14:52 -0500 Subject: [PATCH 330/385] Update nsmix for new thermochem interface. --- examples/nsmix-mpi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/nsmix-mpi.py b/examples/nsmix-mpi.py index 5c704fbd6..6d80eac3c 100644 --- a/examples/nsmix-mpi.py +++ b/examples/nsmix-mpi.py @@ -231,8 +231,8 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, # Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and # generates a set of methods to calculate chemothermomechanical properties and # states for this particular mechanism. - from mirgecom.thermochemistry import make_pyrometheus_mechanism - pyrometheus_mechanism = make_pyrometheus_mechanism(actx, cantera_soln) + from mirgecom.thermochemistry import make_pyrometheus_mechanism_class + pyrometheus_mechanism = make_pyrometheus_mechanism_class(cantera_soln)(actx.np) eos = PyrometheusMixture(pyrometheus_mechanism, temperature_guess=init_temperature, transport_model=transport_model) From 28e67b95cd203672005c3003cde305d0295a6b24 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 26 Oct 2021 07:31:17 -0500 Subject: [PATCH 331/385] Update from upstream --- examples/nsmix-mpi.py | 4 ++-- test/test_eos.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/nsmix-mpi.py b/examples/nsmix-mpi.py index 5bdc8fb97..6d80eac3c 100644 --- a/examples/nsmix-mpi.py +++ b/examples/nsmix-mpi.py @@ -55,7 +55,6 @@ from mirgecom.initializers import MixtureInitializer from mirgecom.eos import PyrometheusMixture import cantera -import pyrometheus as pyro from logpyle import IntervalTimer, set_dt from mirgecom.euler import extract_vars_for_logging, units_for_logging @@ -232,7 +231,8 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, # Create a Pyrometheus EOS with the Cantera soln. Pyrometheus uses Cantera and # generates a set of methods to calculate chemothermomechanical properties and # states for this particular mechanism. - pyrometheus_mechanism = pyro.get_thermochem_class(cantera_soln)(actx.np) + from mirgecom.thermochemistry import make_pyrometheus_mechanism_class + pyrometheus_mechanism = make_pyrometheus_mechanism_class(cantera_soln)(actx.np) eos = PyrometheusMixture(pyrometheus_mechanism, temperature_guess=init_temperature, transport_model=transport_model) diff --git a/test/test_eos.py b/test/test_eos.py index 124b5d6ff..cf144b550 100644 --- a/test/test_eos.py +++ b/test/test_eos.py @@ -41,7 +41,6 @@ as pytest_generate_tests) import cantera -import pyrometheus as pyro from mirgecom.eos import IdealSingleGas, PyrometheusMixture from mirgecom.initializers import ( Vortex2D, Lump, @@ -292,7 +291,9 @@ def test_pyrometheus_kinetics(ctx_factory, mechname, rate_tol, y0): # Pyrometheus initialization mech_cti = get_mechanism_cti(mechname) cantera_soln = cantera.Solution(phase_id="gas", source=mech_cti) - pyro_obj = pyro.get_thermochem_class(cantera_soln)(actx.np) + from mirgecom.thermochemistry import make_pyrometheus_mechanism_class + # pyro_obj = pyro.get_thermochem_class(cantera_soln)(actx.np) + pyro_obj = make_pyrometheus_mechanism_class(cantera_soln)(actx.np) nspecies = pyro_obj.num_species print(f"PrometheusMixture::NumSpecies = {nspecies}") From abe99d15e1b41b5be16f77e545c3d02e7c7f9fc6 Mon Sep 17 00:00:00 2001 From: Matthew Smith Date: Tue, 26 Oct 2021 10:06:22 -0500 Subject: [PATCH 332/385] remove explicit actx argument from advance_state --- mirgecom/steppers.py | 50 ++++++++++++++++++----------------- test/test_time_integrators.py | 6 ++--- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index e14f513bb..3ff5460eb 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -32,10 +32,14 @@ from logpyle import set_dt from mirgecom.logging_quantities import set_sim_state from pytools import memoize_in -from arraycontext import freeze, thaw +from arraycontext import ( + freeze, + thaw, + get_container_context_recursively +) -def compile_timestepper(actx, timestepper, state, rhs): +def _compile_timestepper(actx, timestepper, rhs): """Create lazy evaluation version of the timestepper.""" @memoize_in(actx, ("mirgecom_compiled_operator", timestepper, rhs)) @@ -47,8 +51,11 @@ def get_timestepper(): return get_timestepper() -def compile_rhs(actx, rhs, state): +def _compile_rhs(actx, rhs): """Create lazy evaluation version of the rhs.""" + if actx is None: + return rhs + @memoize_in(actx, ("mirgecom_compiled_rhs", rhs)) def get_rhs(): @@ -57,13 +64,18 @@ def get_rhs(): return get_rhs() +def _evaluate_state(actx, state): + if actx is None: + return state + return thaw(freeze(state, actx), actx) + + def _advance_state_stepper_func(rhs, timestepper, state, t_final, dt=0, t=0.0, istep=0, pre_step_callback=None, post_step_callback=None, - logmgr=None, eos=None, dim=None, - actx=None): + logmgr=None, eos=None, dim=None): """Advance state from some time (t) to some time (t_final). Parameters @@ -109,13 +121,12 @@ def _advance_state_stepper_func(rhs, timestepper, if t_final <= t: return istep, t, state - if actx is None: - actx = state.array_context + actx = get_container_context_recursively(state) - compiled_rhs = compile_rhs(actx, rhs, state) + compiled_rhs = _compile_rhs(actx, rhs) while t < t_final: - state = thaw(freeze(state, actx), actx) + state = _evaluate_state(actx, state) if logmgr: logmgr.tick_before() @@ -145,8 +156,7 @@ def _advance_state_leap(rhs, timestepper, state, t=0.0, istep=0, pre_step_callback=None, post_step_callback=None, - logmgr=None, eos=None, dim=None, - actx=None): + logmgr=None, eos=None, dim=None): """Advance state from some time *t* to some time *t_final* using :mod:`leap`. Parameters @@ -190,19 +200,14 @@ def _advance_state_leap(rhs, timestepper, state, if t_final <= t: return istep, t, state - if actx is None: - actx = state.array_context + actx = get_container_context_recursively(state) - compiled_rhs = compile_rhs(actx, rhs, state) + compiled_rhs = _compile_rhs(actx, rhs) stepper_cls = generate_singlerate_leap_advancer(timestepper, component_id, compiled_rhs, t, dt, state) - while t < t_final: - # This is only needed because Leap testing in test/test_time_integrators.py - # tests on single scalar values rather than an array-context-ready array - # container like a CV. - if isinstance(state, np.ndarray): - state = thaw(freeze(state, actx), actx) + while t < t_final: + state = _evaluate_state(actx, state) if pre_step_callback is not None: state, dt = pre_step_callback(state=state, @@ -271,8 +276,7 @@ def advance_state(rhs, timestepper, state, t_final, t=0.0, istep=0, dt=0, pre_step_callback=None, post_step_callback=None, - logmgr=None, eos=None, dim=None, - actx=None): + logmgr=None, eos=None, dim=None): """Determine what stepper we're using and advance the state from (t) to (t_final). Parameters @@ -347,7 +351,6 @@ def advance_state(rhs, timestepper, state, t_final, post_step_callback=post_step_callback, component_id=component_id, istep=istep, logmgr=logmgr, eos=eos, dim=dim, - actx=actx ) else: (current_step, current_t, current_state) = \ @@ -357,7 +360,6 @@ def advance_state(rhs, timestepper, state, t_final, pre_step_callback=pre_step_callback, post_step_callback=post_step_callback, istep=istep, logmgr=logmgr, eos=eos, dim=dim, - actx=actx ) return current_step, current_t, current_state diff --git a/test/test_time_integrators.py b/test/test_time_integrators.py index a78ac1187..069469778 100644 --- a/test/test_time_integrators.py +++ b/test/test_time_integrators.py @@ -105,10 +105,8 @@ def rhs(t, state): (SSPRK22MethodBuilder("y"), 2), (SSPRK33MethodBuilder("y"), 3), ]) - def test_leapgen_integration_order(actx_factory, method, method_order): + def test_leapgen_integration_order(method, method_order): """Test that time integrators have correct order.""" - actx = actx_factory() - def exact_soln(t): return np.exp(-t) @@ -130,7 +128,7 @@ def rhs(t, y): (step, t, state) = \ advance_state(rhs=rhs, timestepper=method, dt=dt, state=state, t=t, t_final=t_final, - component_id="y", actx=actx) + component_id="y") error = np.abs(state - exact_soln(t)) / exact_soln(t) integrator_eoc.add_data_point(dt, error) From 35191f461ab57e69cb42b7354195d434a96ec099 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Tue, 26 Oct 2021 12:01:25 -0500 Subject: [PATCH 333/385] Switch to arraycontext@main --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 92888638d..0d2540f42 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ pyyaml --editable git+https://github.com/inducer/dagrt.git#egg=dagrt --editable git+https://github.com/inducer/leap.git#egg=leap --editable git+https://github.com/inducer/modepy.git#egg=modepy ---editable git+https://github.com/kaushikcfd/arraycontext.git@pytato-array-context-transforms#egg=arraycontext +--editable git+https://github.com/inducer/arraycontext.git#egg=arraycontext --editable git+https://github.com/kaushikcfd/meshmode.git@pytato-array-context-transforms#egg=meshmode --editable git+https://github.com/inducer/grudge.git#egg=grudge --editable git+https://github.com/kaushikcfd/pytato.git@pytato-array-context-transforms#egg=pytato From 11c64221de4479977a231a4075a84dad0b8bacd4 Mon Sep 17 00:00:00 2001 From: Matthew Smith Date: Thu, 28 Oct 2021 10:50:49 -0500 Subject: [PATCH 334/385] restore grudge branch --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ddfe23291..4dc2b033d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ pyyaml --editable git+https://github.com/inducer/modepy.git#egg=modepy --editable git+https://github.com/inducer/arraycontext.git#egg=arraycontext --editable git+https://github.com/inducer/meshmode.git#egg=meshmode ---editable git+https://github.com/majosm/grudge.git@nodal-reduction-device-scalar#egg=grudge +--editable git+https://github.com/inducer/grudge.git#egg=grudge --editable git+https://github.com/inducer/pytato.git#egg=pytato --editable git+https://github.com/ecisneros8/pyrometheus.git#egg=pyrometheus --editable git+https://github.com/illinois-ceesd/logpyle.git#egg=logpyle From 426e7e2c29658dd2b9483db8c9c1bea630acdea1 Mon Sep 17 00:00:00 2001 From: Matthew Smith Date: Thu, 28 Oct 2021 14:23:46 -0500 Subject: [PATCH 335/385] test production driver changes --- .ci-support/production-testing-env.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 1adeba830..42988ff2e 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -27,3 +27,5 @@ set -x # # Example: # PRODUCTION_DRIVERS="illinois-ceesd/drivers_y1-nozzle@main:w-hagen/isolator@NS" + +PRODUCTION_DRIVERS="illinois-ceesd/drivers_y1-nozzle@main:illinois-ceesd/drivers_y2-isolator@nodal-reduction-device-scalar:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar" From 8a9689bdf702c4377180376103238d1c808bda19 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 28 Oct 2021 14:31:53 -0500 Subject: [PATCH 336/385] Switch nozzle production driver temporarily. --- .ci-support/production-testing-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 42988ff2e..9e578a765 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -28,4 +28,4 @@ set -x # Example: # PRODUCTION_DRIVERS="illinois-ceesd/drivers_y1-nozzle@main:w-hagen/isolator@NS" -PRODUCTION_DRIVERS="illinois-ceesd/drivers_y1-nozzle@main:illinois-ceesd/drivers_y2-isolator@nodal-reduction-device-scalar:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar" +PRODUCTION_DRIVERS="illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@nodal-reduction-device-scalar:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar" From 1ca297833650a2168905fa49c79308c10dd6b1f8 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 28 Oct 2021 15:29:41 -0500 Subject: [PATCH 337/385] Update downstream drivers with device scalars --- examples/doublemach-mpi.py | 8 ++++---- examples/hotplate-mpi.py | 16 ++++++++-------- examples/nsmix-mpi.py | 10 +++++----- examples/poiseuille-mpi.py | 20 ++++++++++---------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/examples/doublemach-mpi.py b/examples/doublemach-mpi.py index aa2969f06..e115fbe32 100644 --- a/examples/doublemach-mpi.py +++ b/examples/doublemach-mpi.py @@ -315,8 +315,8 @@ def my_health_check(state, dv): comm, op=MPI.LOR): health_error = True from grudge.op import nodal_max, nodal_min - p_min = nodal_min(discr, "vol", dv.pressure) - p_max = nodal_max(discr, "vol", dv.pressure) + p_min = actx.to_numpy(nodal_min(discr, "vol", dv.pressure)) + p_max = actx.to_numpy(nodal_max(discr, "vol", dv.pressure)) logger.info(f"Pressure range violation ({p_min=}, {p_max=})") if check_naninf_local(discr, "vol", dv.temperature): @@ -328,8 +328,8 @@ def my_health_check(state, dv): comm, op=MPI.LOR): health_error = True from grudge.op import nodal_max, nodal_min - t_min = nodal_min(discr, "vol", dv.temperature) - t_max = nodal_max(discr, "vol", dv.temperature) + t_min = actx.to_numpy(nodal_min(discr, "vol", dv.temperature)) + t_max = actx.to_numpy(nodal_max(discr, "vol", dv.temperature)) logger.info(f"Temperature range violation ({t_min=}, {t_max=})") return health_error diff --git a/examples/hotplate-mpi.py b/examples/hotplate-mpi.py index 61df85a37..54f3e39ad 100644 --- a/examples/hotplate-mpi.py +++ b/examples/hotplate-mpi.py @@ -251,16 +251,16 @@ def tramp_2d(x_vec, eos, cv=None, **kwargs): def my_write_status(step, t, dt, dv, state, component_errors): from grudge.op import nodal_min, nodal_max - p_min = nodal_min(discr, "vol", dv.pressure) - p_max = nodal_max(discr, "vol", dv.pressure) - t_min = nodal_min(discr, "vol", dv.temperature) - t_max = nodal_max(discr, "vol", dv.temperature) + p_min = actx.to_numpy(nodal_min(discr, "vol", dv.pressure)) + p_max = actx.to_numpy(nodal_max(discr, "vol", dv.pressure)) + t_min = actx.to_numpy(nodal_min(discr, "vol", dv.temperature)) + t_max = actx.to_numpy(nodal_max(discr, "vol", dv.temperature)) if constant_cfl: cfl = current_cfl else: from mirgecom.viscous import get_viscous_cfl - cfl = nodal_max(discr, "vol", - get_viscous_cfl(discr, eos, dt, state)) + cfl = actx.to_numpy(nodal_max(discr, "vol", + get_viscous_cfl(discr, eos, dt, state))) if rank == 0: logger.info(f"Step: {step}, T: {t}, DT: {dt}, CFL: {cfl}\n" f"----- Pressure({p_min}, {p_max})\n" @@ -320,8 +320,8 @@ def my_health_check(state, dv, component_errors): comm, op=MPI.LOR): health_error = True from grudge.op import nodal_max, nodal_min - t_min = nodal_min(discr, "vol", dv.temperature) - t_max = nodal_max(discr, "vol", dv.temperature) + t_min = actx.to_numpy(nodal_min(discr, "vol", dv.temperature)) + t_max = actx.to_numpy(nodal_max(discr, "vol", dv.temperature)) logger.info(f"Temperature range violation ({t_min=}, {t_max=})") exittol = .1 diff --git a/examples/nsmix-mpi.py b/examples/nsmix-mpi.py index 6d80eac3c..1194a45af 100644 --- a/examples/nsmix-mpi.py +++ b/examples/nsmix-mpi.py @@ -309,7 +309,7 @@ def my_write_status(step, t, dt, state): from mirgecom.viscous import get_viscous_cfl cfl_field = get_viscous_cfl(discr, eos, dt, cv=state) from grudge.op import nodal_max - cfl = nodal_max(discr, "vol", cfl_field) + cfl = actx.to_numpy(nodal_max(discr, "vol", cfl_field)) if rank == 0: logger.info(f"Step: {step}, T: {t}, DT: {dt}, CFL: {cfl}") @@ -357,8 +357,8 @@ def my_health_check(state, dv): comm, op=MPI.LOR): health_error = True from grudge.op import nodal_max, nodal_min - p_min = nodal_min(discr, "vol", dv.pressure) - p_max = nodal_max(discr, "vol", dv.pressure) + p_min = actx.to_numpy(nodal_min(discr, "vol", dv.pressure)) + p_max = actx.to_numpy(nodal_max(discr, "vol", dv.pressure)) logger.info(f"Pressure range violation ({p_min=}, {p_max=})") if check_naninf_local(discr, "vol", dv.temperature): @@ -369,8 +369,8 @@ def my_health_check(state, dv): comm, op=MPI.LOR): health_error = True from grudge.op import nodal_max, nodal_min - t_min = nodal_min(discr, "vol", dv.temperature) - t_max = nodal_max(discr, "vol", dv.temperature) + t_min = actx.to_numpy(nodal_min(discr, "vol", dv.temperature)) + t_max = actx.to_numpy(nodal_max(discr, "vol", dv.temperature)) logger.info(f"Temperature range violation ({t_min=}, {t_max=})") return health_error diff --git a/examples/poiseuille-mpi.py b/examples/poiseuille-mpi.py index bfca8ff49..819b3d6cb 100644 --- a/examples/poiseuille-mpi.py +++ b/examples/poiseuille-mpi.py @@ -260,16 +260,16 @@ def poiseuille_2d(x_vec, eos, cv=None, **kwargs): def my_write_status(step, t, dt, dv, state, component_errors): from grudge.op import nodal_min, nodal_max - p_min = nodal_min(discr, "vol", dv.pressure) - p_max = nodal_max(discr, "vol", dv.pressure) - t_min = nodal_min(discr, "vol", dv.temperature) - t_max = nodal_max(discr, "vol", dv.temperature) + p_min = actx.to_numpy(nodal_min(discr, "vol", dv.pressure)) + p_max = actx.to_numpy(nodal_max(discr, "vol", dv.pressure)) + t_min = actx.to_numpy(nodal_min(discr, "vol", dv.temperature)) + t_max = actx.to_numpy(nodal_max(discr, "vol", dv.temperature)) if constant_cfl: cfl = current_cfl else: from mirgecom.viscous import get_viscous_cfl - cfl = nodal_max(discr, "vol", - get_viscous_cfl(discr, eos, dt, state)) + cfl = actx.to_numpy(nodal_max(discr, "vol", + get_viscous_cfl(discr, eos, dt, state))) if rank == 0: logger.info(f"Step: {step}, T: {t}, DT: {dt}, CFL: {cfl}\n" f"----- Pressure({p_min}, {p_max})\n" @@ -317,8 +317,8 @@ def my_health_check(state, dv, component_errors): comm, op=MPI.LOR): health_error = True from grudge.op import nodal_max, nodal_min - p_min = nodal_min(discr, "vol", dv.pressure) - p_max = nodal_max(discr, "vol", dv.pressure) + p_min = actx.to_numpy(nodal_min(discr, "vol", dv.pressure)) + p_max = actx.to_numpy(nodal_max(discr, "vol", dv.pressure)) logger.info(f"Pressure range violation ({p_min=}, {p_max=})") if check_naninf_local(discr, "vol", dv.temperature): @@ -329,8 +329,8 @@ def my_health_check(state, dv, component_errors): comm, op=MPI.LOR): health_error = True from grudge.op import nodal_max, nodal_min - t_min = nodal_min(discr, "vol", dv.temperature) - t_max = nodal_max(discr, "vol", dv.temperature) + t_min = actx.to_numpy(nodal_min(discr, "vol", dv.temperature)) + t_max = actx.to_numpy(nodal_max(discr, "vol", dv.temperature)) logger.info(f"Temperature range violation ({t_min=}, {t_max=})") exittol = .1 From 32aa4fdf44f0b6f68b17375d0b2f20894051b132 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 28 Oct 2021 17:37:20 -0500 Subject: [PATCH 338/385] Use *this* branch as production. --- .ci-support/production-testing-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 9e578a765..0e22f8761 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -15,7 +15,7 @@ set -x # patched by the incoming development. The following vars control the # production environment: # -# export PRODUCTION_BRANCH="" # The base production branch to be installed by emirge +export PRODUCTION_BRANCH="y2-production" # The base production branch to be installed by emirge # export PRODUCTION_FORK="" # The fork/home of production changes (if any) # # Multiple production drivers are supported. The user should provide a ':'-delimited From aaf773478238355087f2d482de4bc84a4793cbf9 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 28 Oct 2021 18:02:49 -0500 Subject: [PATCH 339/385] Source the env file. --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3d0ff1d31..4720738d9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -171,6 +171,7 @@ jobs: cd .. git clone https://github.com/illinois-ceesd/emirge emirge.y1 cd emirge.y1 + . ../mirgecom/.ci_support/production-testing-env.sh . ../mirgecom/.ci-support/production-install.sh ../mirgecom/.ci-support/production-testing-env.sh - name: Run production test run: | From a97529fc2cc01c984132d148ed272ca331bf6c7d Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 28 Oct 2021 18:14:53 -0500 Subject: [PATCH 340/385] Correct typo --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4720738d9..db21db82d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -171,7 +171,7 @@ jobs: cd .. git clone https://github.com/illinois-ceesd/emirge emirge.y1 cd emirge.y1 - . ../mirgecom/.ci_support/production-testing-env.sh + . ../mirgecom/.ci-support/production-testing-env.sh . ../mirgecom/.ci-support/production-install.sh ../mirgecom/.ci-support/production-testing-env.sh - name: Run production test run: | From c379efac68e1783c55646feb73a622b9a67fdfc0 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 28 Oct 2021 21:08:54 -0500 Subject: [PATCH 341/385] use to_numpy where approp --- test/test_navierstokes.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/test/test_navierstokes.py b/test/test_navierstokes.py index 0f2e18ba5..45d52897f 100644 --- a/test/test_navierstokes.py +++ b/test/test_navierstokes.py @@ -138,14 +138,14 @@ def test_uniform_rhs(actx_factory, nspecies, dim, order): f"rhoy_rhs = {rhoy_rhs}\n" ) - assert discr.norm(rho_resid, np.inf) < tolerance - assert discr.norm(rhoe_resid, np.inf) < tolerance + assert actx.to_numpy(discr.norm(rho_resid, np.inf)) < tolerance + assert actx.to_numpy(discr.norm(rhoe_resid, np.inf)) < tolerance for i in range(dim): - assert discr.norm(mom_resid[i], np.inf) < tolerance + assert actx.to_numpy(discr.norm(mom_resid[i], np.inf)) < tolerance for i in range(nspecies): - assert discr.norm(rhoy_resid[i], np.inf) < tolerance + assert actx.to_numpy(discr.norm(rhoy_resid[i], np.inf)) < tolerance - err_max = discr.norm(rho_resid, np.inf) + err_max = actx.to_numpy(discr.norm(rho_resid, np.inf)) eoc_rec0.add_data_point(1.0 / nel_1d, err_max) # set a non-zero, but uniform velocity component @@ -165,15 +165,15 @@ def test_uniform_rhs(actx_factory, nspecies, dim, order): mom_resid = rhs_resid.momentum rhoy_resid = rhs_resid.species_mass - assert discr.norm(rho_resid, np.inf) < tolerance - assert discr.norm(rhoe_resid, np.inf) < tolerance + assert actx.to_numpy(discr.norm(rho_resid, np.inf)) < tolerance + assert actx.to_numpy(discr.norm(rhoe_resid, np.inf)) < tolerance for i in range(dim): - assert discr.norm(mom_resid[i], np.inf) < tolerance + assert actx.to_numpy(discr.norm(mom_resid[i], np.inf)) < tolerance for i in range(nspecies): - assert discr.norm(rhoy_resid[i], np.inf) < tolerance + assert actx.to_numpy(discr.norm(rhoy_resid[i], np.inf)) < tolerance - err_max = discr.norm(rho_resid, np.inf) + err_max = actx.to_numpy(discr.norm(rho_resid, np.inf)) eoc_rec1.add_data_point(1.0 / nel_1d, err_max) logger.info( @@ -310,14 +310,15 @@ def poiseuille_2d(x_vec, eos, cv=None, **kwargs): ) tol_fudge = 2e-4 - assert discr.norm(rho_resid, np.inf) < tolerance - # assert discr.norm(rhoe_resid, np.inf) < tolerance - mom_err = [discr.norm(mom_resid[i], np.inf) for i in range(dim)] + assert actx.to_numpy(discr.norm(rho_resid, np.inf)) < tolerance + # assert actx.to_numpy(discr.norm(rhoe_resid, np.inf)) < tolerance + mom_err = [actx.to_numpy(discr.norm(mom_resid[i], np.inf)) + for i in range(dim)] err_max = max(mom_err) for i in range(dim): assert mom_err[i] < tol_fudge - # err_max = discr.norm(rho_resid, np.inf) + # err_max = actx.to_numpy(discr.norm(rho_resid, np.inf) eoc_rec.add_data_point(1.0 / nfac, err_max) logger.info( From 6464542a70dd8b95f0c4529fe1db21031a2124b1 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 28 Oct 2021 21:45:28 -0500 Subject: [PATCH 342/385] Use Y2-production branch for isolator driver --- .ci-support/production-testing-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 0e22f8761..4c960cf7e 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -28,4 +28,4 @@ export PRODUCTION_BRANCH="y2-production" # The base production branch to be in # Example: # PRODUCTION_DRIVERS="illinois-ceesd/drivers_y1-nozzle@main:w-hagen/isolator@NS" -PRODUCTION_DRIVERS="illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@nodal-reduction-device-scalar:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar" +PRODUCTION_DRIVERS="illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@y2-production:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar" From 563cb0ffd9fb3f9b67810528a8e78596dff814e2 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Thu, 28 Oct 2021 22:21:26 -0500 Subject: [PATCH 343/385] Revert the CI settings to new drivers and empty customization --- .ci-support/production-driver-install.sh | 8 ++++---- .ci-support/production-testing-env.sh | 4 +--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.ci-support/production-driver-install.sh b/.ci-support/production-driver-install.sh index ff9810a9c..fae77f427 100644 --- a/.ci-support/production-driver-install.sh +++ b/.ci-support/production-driver-install.sh @@ -9,9 +9,9 @@ # PRODUCTION_DRIVERS = ':' delimited list "fork/repo@branch" # (See the example default value below) # -# The default values result in an install of the Y1 nozzle driver and -# Wyatt Hagen's isolator driver that work with current MIRGE-Com -# production branch: mirgecom@y1-production. +# The default values result in an install of the Y1 nozzle driver, +# a 1dflame driver, and the Y2 isolator driver that work with current +# MIRGE-Com production branch: mirgecom@y1-production. # set -x @@ -24,7 +24,7 @@ if [ -n "$DEVELOPMENT_BRANCH" ]; then fi fi # Set to default if testing main, or user left it empty -PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@main:w-hagen/isolator@master:illinois-ceesd/drivers_flame1d@main"} +PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@y2-production:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar"} OIFS="$IFS" IFS=':'; for production_driver_string in $PRODUCTION_DRIVERS; do diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 4c960cf7e..1adeba830 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -15,7 +15,7 @@ set -x # patched by the incoming development. The following vars control the # production environment: # -export PRODUCTION_BRANCH="y2-production" # The base production branch to be installed by emirge +# export PRODUCTION_BRANCH="" # The base production branch to be installed by emirge # export PRODUCTION_FORK="" # The fork/home of production changes (if any) # # Multiple production drivers are supported. The user should provide a ':'-delimited @@ -27,5 +27,3 @@ export PRODUCTION_BRANCH="y2-production" # The base production branch to be in # # Example: # PRODUCTION_DRIVERS="illinois-ceesd/drivers_y1-nozzle@main:w-hagen/isolator@NS" - -PRODUCTION_DRIVERS="illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@y2-production:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar" From 6bd03c7af59e75de3697d51f7676702502e1485a Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Thu, 28 Oct 2021 23:22:44 -0500 Subject: [PATCH 344/385] Account for grudge nodal reductions returning device scalars (#518) Co-authored-by: Mike Campbell --- .ci-support/production-driver-install.sh | 2 +- .ci-support/production-testing-env.sh | 1 + .github/workflows/ci.yaml | 1 + examples/autoignition-mpi.py | 4 +- examples/heat-source-mpi.py | 2 +- examples/vortex-mpi.py | 6 ++- examples/wave-mpi.py | 8 ++-- examples/wave.py | 6 +-- mirgecom/logging_quantities.py | 5 +- mirgecom/simutil.py | 16 ++++--- test/test_bc.py | 9 ++-- test/test_diffusion.py | 10 ++-- test/test_eos.py | 61 +++++++++++++++--------- test/test_euler.py | 34 +++++++------ test/test_filter.py | 6 +-- test/test_fluid.py | 22 ++++++--- test/test_init.py | 38 +++++++++------ test/test_inviscid.py | 26 ++++++---- test/test_lazy.py | 4 +- test/test_operators.py | 13 +++-- test/test_restart.py | 2 +- test/test_simutil.py | 2 +- test/test_symbolic.py | 2 +- test/test_viscous.py | 51 ++++++++++++-------- test/test_wave.py | 9 ++-- 25 files changed, 207 insertions(+), 133 deletions(-) diff --git a/.ci-support/production-driver-install.sh b/.ci-support/production-driver-install.sh index ff9810a9c..e01c71aca 100644 --- a/.ci-support/production-driver-install.sh +++ b/.ci-support/production-driver-install.sh @@ -24,7 +24,7 @@ if [ -n "$DEVELOPMENT_BRANCH" ]; then fi fi # Set to default if testing main, or user left it empty -PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@main:w-hagen/isolator@master:illinois-ceesd/drivers_flame1d@main"} +PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@y2-production:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar"} OIFS="$IFS" IFS=':'; for production_driver_string in $PRODUCTION_DRIVERS; do diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 1adeba830..6cf17aa41 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -10,6 +10,7 @@ set -x # forks, the environment config files should set: # # export DEVELOPMENT_FORK="" # the fork/home of the development +# export DEVELOPMENT_BRANCH="" # # The production capability to test against may be specified outright, or # patched by the incoming development. The following vars control the diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3d0ff1d31..db21db82d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -171,6 +171,7 @@ jobs: cd .. git clone https://github.com/illinois-ceesd/emirge emirge.y1 cd emirge.y1 + . ../mirgecom/.ci-support/production-testing-env.sh . ../mirgecom/.ci-support/production-install.sh ../mirgecom/.ci-support/production-testing-env.sh - name: Run production test run: | diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index a60dda8e1..74483a271 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -412,13 +412,13 @@ def my_get_timestep(t, dt, state): from mirgecom.inviscid import get_inviscid_timestep ts_field = current_cfl * get_inviscid_timestep(discr, eos=eos, cv=state) from grudge.op import nodal_min - dt = nodal_min(discr, "vol", ts_field) + dt = actx.to_numpy(nodal_min(discr, "vol", ts_field))[()] cfl = current_cfl else: from mirgecom.inviscid import get_inviscid_cfl ts_field = get_inviscid_cfl(discr, eos=eos, dt=dt, cv=state) from grudge.op import nodal_max - cfl = nodal_max(discr, "vol", ts_field) + cfl = actx.to_numpy(nodal_max(discr, "vol", ts_field))[()] return ts_field, cfl, min(t_remaining, dt) diff --git a/examples/heat-source-mpi.py b/examples/heat-source-mpi.py index deeee6e1f..30e909112 100644 --- a/examples/heat-source-mpi.py +++ b/examples/heat-source-mpi.py @@ -182,7 +182,7 @@ def rhs(t, u): if logmgr: set_dt(logmgr, dt) logmgr.tick_after() - final_answer = discr.norm(u, np.inf) + final_answer = actx.to_numpy(discr.norm(u, np.inf)) resid = abs(final_answer - 0.00020620711665201585) if resid > 1e-15: raise ValueError(f"Run did not produce the expected result {resid=}") diff --git a/examples/vortex-mpi.py b/examples/vortex-mpi.py index b31e28cf4..fc5071029 100644 --- a/examples/vortex-mpi.py +++ b/examples/vortex-mpi.py @@ -227,8 +227,10 @@ def my_write_status(state, component_errors, cfl=None): else: from grudge.op import nodal_max from mirgecom.inviscid import get_inviscid_cfl - cfl = nodal_max(discr, "vol", - get_inviscid_cfl(discr, eos, current_dt, cv=state)) + cfl = actx.to_numpy( + nodal_max( + discr, "vol", + get_inviscid_cfl(discr, eos, current_dt, cv=state)))[()] if rank == 0: logger.info( f"------ {cfl=}\n" diff --git a/examples/wave-mpi.py b/examples/wave-mpi.py index 267ec9d9f..f7ba0f662 100644 --- a/examples/wave-mpi.py +++ b/examples/wave-mpi.py @@ -138,10 +138,10 @@ def main(snapshot_pattern="wave-mpi-{step:04d}-{rank:04d}.pkl", restart_step=Non current_cfl = 0.485 wave_speed = 1.0 from grudge.dt_utils import characteristic_lengthscales - dt = current_cfl * characteristic_lengthscales(actx, discr) / wave_speed + nodal_dt = characteristic_lengthscales(actx, discr) / wave_speed from grudge.op import nodal_min - dt = nodal_min(discr, "vol", dt) + dt = actx.to_numpy(current_cfl * nodal_min(discr, "vol", nodal_dt))[()] t_final = 1 @@ -217,7 +217,7 @@ def rhs(t, w): ) if istep % 10 == 0: - print(istep, t, discr.norm(fields[0])) + print(istep, t, actx.to_numpy(discr.norm(fields[0]))) vis.write_parallel_vtk_file( comm, "fld-wave-mpi-%03d-%04d.vtu" % (rank, istep), @@ -237,7 +237,7 @@ def rhs(t, w): set_dt(logmgr, dt) logmgr.tick_after() - final_soln = discr.norm(fields[0]) + final_soln = actx.to_numpy(discr.norm(fields[0])) assert np.abs(final_soln - 0.04409852463947439) < 1e-14 diff --git a/examples/wave.py b/examples/wave.py index b652b42ae..27d922a40 100644 --- a/examples/wave.py +++ b/examples/wave.py @@ -106,9 +106,9 @@ def main(use_profiling=False, use_logmgr=False, lazy_eval: bool = False): current_cfl = 0.485 wave_speed = 1.0 from grudge.dt_utils import characteristic_lengthscales - dt = current_cfl * characteristic_lengthscales(actx, discr) / wave_speed + nodal_dt = characteristic_lengthscales(actx, discr) / wave_speed from grudge.op import nodal_min - dt = nodal_min(discr, "vol", dt) + dt = actx.to_numpy(current_cfl * nodal_min(discr, "vol", nodal_dt))[()] print("%d elements" % mesh.nelements) @@ -152,7 +152,7 @@ def rhs(t, w): if istep % 10 == 0: if use_profiling: print(actx.tabulate_profiling_data()) - print(istep, t, discr.norm(fields[0], np.inf)) + print(istep, t, actx.to_numpy(discr.norm(fields[0], np.inf))) vis.write_vtk_file("fld-wave-%04d.vtu" % istep, [ ("u", fields[0]), diff --git a/mirgecom/logging_quantities.py b/mirgecom/logging_quantities.py index 5e94e1965..40dd2161f 100644 --- a/mirgecom/logging_quantities.py +++ b/mirgecom/logging_quantities.py @@ -42,6 +42,7 @@ from logpyle import (LogQuantity, PostLogQuantity, LogManager, MultiPostLogQuantity, add_run_info, add_general_quantities, add_simulation_quantities) +from arraycontext.container import get_container_context_recursively from meshmode.array_context import PyOpenCLArrayContext from meshmode.discretization import Discretization import pyopencl as cl @@ -279,10 +280,12 @@ def __call__(self): quantity = self.state_vars[self.quantity] + actx = get_container_context_recursively(quantity) + if self.axis is not None: # e.g. momentum quantity = quantity[self.axis] - return self._discr_reduction(quantity) + return actx.to_numpy(self._discr_reduction(quantity))[()] # }}} diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index d94cd3916..45f1cf6e1 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -119,10 +119,10 @@ def get_sim_timestep(discr, state, t, dt, cfl, eos, if constant_cfl: from mirgecom.viscous import get_viscous_timestep from grudge.op import nodal_min - mydt = cfl * nodal_min( - discr, "vol", - get_viscous_timestep(discr=discr, eos=eos, cv=state) - ) + mydt = state.array_context.to_numpy( + cfl * nodal_min( + discr, "vol", + get_viscous_timestep(discr=discr, eos=eos, cv=state)))[()] return min(t_remaining, mydt) @@ -269,9 +269,10 @@ def allsync(local_values, comm=None, op=None): def check_range_local(discr, dd, field, min_value, max_value): """Check for any negative values.""" + actx = field.array_context return ( - op.nodal_min_loc(discr, dd, field) < min_value - or op.nodal_max_loc(discr, dd, field) > max_value + actx.to_numpy(op.nodal_min_loc(discr, dd, field)) < min_value + or actx.to_numpy(op.nodal_max_loc(discr, dd, field)) > max_value ) @@ -288,8 +289,9 @@ def compare_fluid_solutions(discr, red_state, blue_state): .. note:: This is a collective routine and must be called by all MPI ranks. """ + actx = red_state.array_context resid = red_state - blue_state - return [discr.norm(v, np.inf) for v in resid.join()] + return [actx.to_numpy(discr.norm(v, np.inf)) for v in resid.join()] def generate_and_distribute_mesh(comm, generate_mesh): diff --git a/test/test_bc.py b/test/test_bc.py index 3ef0894c9..133edc95e 100644 --- a/test/test_bc.py +++ b/test/test_bc.py @@ -83,8 +83,9 @@ def test_slipwall_identity(actx_factory, dim): wall = AdiabaticSlipBoundary() uniform_state = initializer(nodes) - from functools import partial - bnd_norm = partial(discr.norm, p=np.inf, dd=BTAG_ALL) + + def bnd_norm(vec): + return actx.to_numpy(discr.norm(vec, p=np.inf, dd=BTAG_ALL)) bnd_pair = wall.boundary_pair(discr, btag=BTAG_ALL, eos=eos, cv=uniform_state) @@ -135,8 +136,8 @@ def test_slipwall_flux(actx_factory, dim, order): nhat = thaw(actx, discr.normal(BTAG_ALL)) h = 1.0 / nel_1d - from functools import partial - bnd_norm = partial(discr.norm, p=np.inf, dd=BTAG_ALL) + def bnd_norm(vec): + return actx.to_numpy(discr.norm(vec, p=np.inf, dd=BTAG_ALL)) logger.info(f"Number of {dim}d elems: {mesh.nelements}") # for velocities in each direction diff --git a/test/test_diffusion.py b/test/test_diffusion.py index e66b06293..7a74785d4 100644 --- a/test/test_diffusion.py +++ b/test/test_diffusion.py @@ -301,7 +301,7 @@ def get_rhs(t, u): expected_u = sym_eval(p.sym_u, t) - rel_linf_err = ( + rel_linf_err = actx.to_numpy( discr.norm(u - expected_u, np.inf) / discr.norm(expected_u, np.inf)) eoc_rec.add_data_point(1./n, rel_linf_err) @@ -385,7 +385,7 @@ def get_rhs(t, u): ("rhs", rhs), ]) - linf_err = discr.norm(rhs, np.inf) + linf_err = actx.to_numpy(discr.norm(rhs, np.inf)) assert(linf_err < 1e-11) # Now check stability @@ -415,7 +415,7 @@ def get_rhs(t, u): ("u_steady", u_steady), ]) - linf_diff = discr.norm(u - u_steady, np.inf) + linf_diff = actx.to_numpy(discr.norm(u - u_steady, np.inf)) assert linf_diff < 0.1 @@ -536,7 +536,7 @@ def sym_eval(expr): assert isinstance(diffusion_u1, DOFArray) expected_diffusion_u1 = sym_eval(sym_diffusion_u1) - rel_linf_err = ( + rel_linf_err = actx.to_numpy( discr.norm(diffusion_u1 - expected_diffusion_u1, np.inf) / discr.norm(expected_diffusion_u1, np.inf)) assert rel_linf_err < 1.e-5 @@ -556,7 +556,7 @@ def sym_eval(expr): sym_eval(sym_diffusion_u1), sym_eval(sym_diffusion_u2) ]) - rel_linf_err = ( + rel_linf_err = actx.to_numpy( discr.norm(diffusion_u_vector - expected_diffusion_u_vector, np.inf) / discr.norm(expected_diffusion_u_vector, np.inf)) assert rel_linf_err < 1.e-5 diff --git a/test/test_eos.py b/test/test_eos.py index 124b5d6ff..57cec7f40 100644 --- a/test/test_eos.py +++ b/test/test_eos.py @@ -150,18 +150,21 @@ def test_pyrometheus_mechanisms(ctx_factory, mechname, rate_tol, y0): print(f"can_omega = {can_omega}") print(f"prom_omega = {prom_omega}") - assert discr.norm((prom_c - can_c) / can_c, np.inf) < 1e-14 - assert discr.norm((prom_t - can_t) / can_t, np.inf) < 1e-14 - assert discr.norm((prom_rho - can_rho) / can_rho, np.inf) < 1e-14 - assert discr.norm((prom_p - can_p) / can_p, np.inf) < 1e-14 - assert discr.norm((prom_e - can_e) / can_e, np.inf) < 1e-6 - assert discr.norm((prom_k - can_k) / can_k, np.inf) < 1e-10 + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + assert inf_norm((prom_c - can_c) / can_c) < 1e-14 + assert inf_norm((prom_t - can_t) / can_t) < 1e-14 + assert inf_norm((prom_rho - can_rho) / can_rho) < 1e-14 + assert inf_norm((prom_p - can_p) / can_p) < 1e-14 + assert inf_norm((prom_e - can_e) / can_e) < 1e-6 + assert inf_norm((prom_k - can_k) / can_k) < 1e-10 # Pyro chem test comparisons for i, rate in enumerate(can_r): - assert discr.norm((prom_r[i] - rate), np.inf) < rate_tol + assert inf_norm(prom_r[i] - rate) < rate_tol for i, rate in enumerate(can_omega): - assert discr.norm((prom_omega[i] - rate), np.inf) < rate_tol + assert inf_norm(prom_omega[i] - rate) < rate_tol @pytest.mark.parametrize("mechname", ["uiuc", "sanDiego"]) @@ -248,11 +251,14 @@ def test_pyrometheus_eos(ctx_factory, mechname, dim, y0, vel): print(f"pyro_eos.temp = {temperature}") print(f"pyro_eos.e = {internal_energy}") + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + tol = 1e-14 - assert discr.norm((cv.mass - pyro_rho) / pyro_rho, np.inf) < tol - assert discr.norm((temperature - pyro_t) / pyro_t, np.inf) < tol - assert discr.norm((internal_energy - pyro_e) / pyro_e, np.inf) < tol - assert discr.norm((p - pyro_p) / pyro_p, np.inf) < tol + assert inf_norm((cv.mass - pyro_rho) / pyro_rho) < tol + assert inf_norm((temperature - pyro_t) / pyro_t) < tol + assert inf_norm((internal_energy - pyro_e) / pyro_e) < tol + assert inf_norm((p - pyro_p) / pyro_p) < tol @pytest.mark.parametrize(("mechname", "rate_tol"), @@ -347,24 +353,27 @@ def test_pyrometheus_kinetics(ctx_factory, mechname, rate_tol, y0): pyro_omega = pyro_obj.get_net_production_rates(rhoin, tin, yin) # Print + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + print(f"can_r = {can_r}") print(f"pyro_r = {pyro_r}") - abs_diff = discr.norm(pyro_r - can_r, np.inf) + abs_diff = inf_norm(pyro_r - can_r) if abs_diff > 1e-14: min_r = (np.abs(can_r)).min() if min_r > 0: - assert discr.norm((pyro_r - can_r) / can_r, np.inf) < rate_tol + assert inf_norm((pyro_r - can_r) / can_r) < rate_tol else: - assert discr.norm(pyro_r, np.inf) < rate_tol + assert inf_norm(pyro_r) < rate_tol print(f"can_omega = {can_omega}") print(f"pyro_omega = {pyro_omega}") for i, omega in enumerate(can_omega): omin = np.abs(omega).min() if omin > 1e-12: - assert discr.norm((pyro_omega[i] - omega) / omega, np.inf) < 1e-8 + assert inf_norm((pyro_omega[i] - omega) / omega) < 1e-8 else: - assert discr.norm(pyro_omega[i], np.inf) < 1e-12 + assert inf_norm(pyro_omega[i]) < 1e-12 @pytest.mark.parametrize("dim", [1, 2, 3]) @@ -400,16 +409,19 @@ def test_idealsingle_lump(ctx_factory, dim): eos = IdealSingleGas() cv = lump(nodes) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + p = eos.pressure(cv) exp_p = 1.0 - errmax = discr.norm(p - exp_p, np.inf) + errmax = inf_norm(p - exp_p) exp_ke = 0.5 * cv.mass ke = eos.kinetic_energy(cv) - kerr = discr.norm(ke - exp_ke, np.inf) + kerr = inf_norm(ke - exp_ke) te = eos.total_energy(cv, p) - terr = discr.norm(te - cv.energy, np.inf) + terr = inf_norm(te - cv.energy) logger.info(f"lump_soln = {cv}") logger.info(f"pressure = {p}") @@ -448,17 +460,20 @@ def test_idealsingle_vortex(ctx_factory): vortex = Vortex2D() cv = vortex(nodes) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + gamma = eos.gamma() p = eos.pressure(cv) exp_p = cv.mass ** gamma - errmax = discr.norm(p - exp_p, np.inf) + errmax = inf_norm(p - exp_p) exp_ke = 0.5 * np.dot(cv.momentum, cv.momentum) / cv.mass ke = eos.kinetic_energy(cv) - kerr = discr.norm(ke - exp_ke, np.inf) + kerr = inf_norm(ke - exp_ke) te = eos.total_energy(cv, p) - terr = discr.norm(te - cv.energy, np.inf) + terr = inf_norm(te - cv.energy) logger.info(f"vortex_soln = {cv}") logger.info(f"pressure = {p}") diff --git a/test/test_euler.py b/test/test_euler.py index 1f7b3daa6..821e9b43e 100644 --- a/test/test_euler.py +++ b/test/test_euler.py @@ -137,14 +137,17 @@ def test_uniform_rhs(actx_factory, nspecies, dim, order): f"rhoy_rhs = {rhoy_rhs}\n" ) - assert discr.norm(rho_resid, np.inf) < tolerance - assert discr.norm(rhoe_resid, np.inf) < tolerance + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + assert inf_norm(rho_resid) < tolerance + assert inf_norm(rhoe_resid) < tolerance for i in range(dim): - assert discr.norm(mom_resid[i], np.inf) < tolerance + assert inf_norm(mom_resid[i]) < tolerance for i in range(nspecies): - assert discr.norm(rhoy_resid[i], np.inf) < tolerance + assert inf_norm(rhoy_resid[i]) < tolerance - err_max = discr.norm(rho_resid, np.inf) + err_max = inf_norm(rho_resid) eoc_rec0.add_data_point(1.0 / nel_1d, err_max) # set a non-zero, but uniform velocity component @@ -165,15 +168,15 @@ def test_uniform_rhs(actx_factory, nspecies, dim, order): mom_resid = rhs_resid.momentum rhoy_resid = rhs_resid.species_mass - assert discr.norm(rho_resid, np.inf) < tolerance - assert discr.norm(rhoe_resid, np.inf) < tolerance + assert inf_norm(rho_resid) < tolerance + assert inf_norm(rhoe_resid) < tolerance for i in range(dim): - assert discr.norm(mom_resid[i], np.inf) < tolerance + assert inf_norm(mom_resid[i]) < tolerance for i in range(nspecies): - assert discr.norm(rhoy_resid[i], np.inf) < tolerance + assert inf_norm(rhoy_resid[i]) < tolerance - err_max = discr.norm(rho_resid, np.inf) + err_max = inf_norm(rho_resid) eoc_rec1.add_data_point(1.0 / nel_1d, err_max) logger.info( @@ -231,7 +234,7 @@ def test_vortex_rhs(actx_factory, order): discr, eos=IdealSingleGas(), boundaries=boundaries, cv=vortex_soln, time=0.0) - err_max = discr.norm(inviscid_rhs.join(), np.inf) + err_max = actx.to_numpy(discr.norm(inviscid_rhs.join(), np.inf)) eoc_rec.add_data_point(1.0 / nel_1d, err_max) logger.info( @@ -290,7 +293,8 @@ def test_lump_rhs(actx_factory, dim, order): ) expected_rhs = lump.exact_rhs(discr, cv=lump_soln, time=0) - err_max = discr.norm((inviscid_rhs-expected_rhs).join(), np.inf) + err_max = actx.to_numpy( + discr.norm((inviscid_rhs-expected_rhs).join(), np.inf)) if err_max > maxxerr: maxxerr = err_max @@ -365,7 +369,9 @@ def test_multilump_rhs(actx_factory, dim, order, v0): print(f"inviscid_rhs = {inviscid_rhs}") print(f"expected_rhs = {expected_rhs}") - err_max = discr.norm((inviscid_rhs-expected_rhs).join(), np.inf) + + err_max = actx.to_numpy( + discr.norm((inviscid_rhs-expected_rhs).join(), np.inf)) if err_max > maxxerr: maxxerr = err_max @@ -500,7 +506,7 @@ def rhs(t, q): maxerr = max(write_soln(cv, False)) else: expected_result = initializer(nodes, time=t) - maxerr = discr.norm((cv - expected_result).join(), np.inf) + maxerr = actx.to_numpy(discr.norm((cv - expected_result).join(), np.inf)) logger.info(f"Max Error: {maxerr}") if maxerr > exittol: diff --git a/test/test_filter.py b/test/test_filter.py index 78b7d5028..85814fb9c 100644 --- a/test/test_filter.py +++ b/test/test_filter.py @@ -192,7 +192,7 @@ def test_filter_function(actx_factory, dim, order, do_viz=False): filtered_soln = filter_modally(discr, "vol", cutoff, frfunc, uniform_soln) soln_resid = uniform_soln - filtered_soln - max_errors = [discr.norm(v, np.inf) for v in soln_resid] + max_errors = [actx.to_numpy(discr.norm(v, np.inf)) for v in soln_resid] tol = 1e-14 @@ -217,7 +217,7 @@ def polyfn(coeff): # , x_vec): filtered_field = filter_modally(discr, "vol", cutoff, frfunc, field) soln_resid = field - filtered_field - max_errors = [discr.norm(v, np.inf) for v in soln_resid] + max_errors = [actx.to_numpy(discr.norm(v, np.inf)) for v in soln_resid] logger.info(f"Field = {field}") logger.info(f"Filtered = {filtered_field}") logger.info(f"Max Errors (poly) = {max_errors}") @@ -253,6 +253,6 @@ def polyfn(coeff): # , x_vec): ] vis.write_vtk_file(f"filter_test_{field_order}.vtu", io_fields) field_resid = unfiltered_spectrum - filtered_spectrum - max_errors = [discr.norm(v, np.inf) for v in field_resid] + max_errors = [actx.to_numpy(discr.norm(v, np.inf)) for v in field_resid] # fields should be different, but not too different assert(tol > np.max(max_errors) > threshold) diff --git a/test/test_fluid.py b/test/test_fluid.py index 4f24e0581..9cac5d7aa 100644 --- a/test/test_fluid.py +++ b/test/test_fluid.py @@ -84,7 +84,7 @@ def test_velocity_gradient_sanity(actx_factory, dim, mass_exp, vel_fac): tol = 1e-11 exp_result = vel_fac * np.eye(dim) * ones - grad_v_err = [discr.norm(grad_v[i] - exp_result[i], np.inf) + grad_v_err = [actx.to_numpy(discr.norm(grad_v[i] - exp_result[i], np.inf)) for i in range(dim)] assert max(grad_v_err) < tol @@ -133,7 +133,8 @@ def exact_grad_row(xdata, gdim, dim): return exact_grad_row comp_err = make_obj_array([ - discr.norm(grad_v[i] - exact_grad_row(nodes[i], i, dim), np.inf) + actx.to_numpy( + discr.norm(grad_v[i] - exact_grad_row(nodes[i], i, dim), np.inf)) for i in range(dim)]) err_max = comp_err.max() eoc.add_data_point(h, err_max) @@ -188,9 +189,13 @@ def test_velocity_gradient_structure(actx_factory): assert grad_v.shape == (dim, dim) from meshmode.dof_array import DOFArray assert type(grad_v[0, 0]) == DOFArray - assert discr.norm(grad_v - exp_result, np.inf) < tol - assert discr.norm(grad_v.T - exp_trans, np.inf) < tol - assert discr.norm(np.trace(grad_v) - exp_trace, np.inf) < tol + + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + assert inf_norm(grad_v - exp_result) < tol + assert inf_norm(grad_v.T - exp_trans) < tol + assert inf_norm(np.trace(grad_v) - exp_trace) < tol @pytest.mark.parametrize("dim", [1, 2, 3]) @@ -237,10 +242,13 @@ def test_species_mass_gradient(actx_factory, dim): from meshmode.dof_array import DOFArray assert type(grad_y[0, 0]) == DOFArray + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + tol = 1e-11 for idim in range(dim): ispec = 2*idim exact_grad = np.array([(ispec*(idim*dim+1))*(iidim+1) for iidim in range(dim)]) - assert discr.norm(grad_y[ispec] - exact_grad, np.inf) < tol - assert discr.norm(grad_y[ispec+1] + exact_grad, np.inf) < tol + assert inf_norm(grad_y[ispec] - exact_grad) < tol + assert inf_norm(grad_y[ispec+1] + exact_grad) < tol diff --git a/test/test_init.py b/test/test_init.py index d2d85af99..c65c445db 100644 --- a/test/test_init.py +++ b/test/test_init.py @@ -84,7 +84,7 @@ def test_uniform_init(ctx_factory, dim, nspecies): def inf_norm(data): if len(data) > 0: - return discr.norm(data, np.inf) + return actx.to_numpy(discr.norm(data, np.inf)) else: return 0.0 @@ -140,7 +140,7 @@ def test_lump_init(ctx_factory): p = 0.4 * (cv.energy - 0.5 * np.dot(cv.momentum, cv.momentum) / cv.mass) exp_p = 1.0 - errmax = discr.norm(p - exp_p, np.inf) + errmax = actx.to_numpy(discr.norm(p - exp_p, np.inf)) logger.info(f"lump_soln = {cv}") logger.info(f"pressure = {p}") @@ -177,7 +177,7 @@ def test_vortex_init(ctx_factory): gamma = 1.4 p = 0.4 * (cv.energy - 0.5 * np.dot(cv.momentum, cv.momentum) / cv.mass) exp_p = cv.mass ** gamma - errmax = discr.norm(p - exp_p, np.inf) + errmax = actx.to_numpy(discr.norm(p - exp_p, np.inf)) logger.info(f"vortex_soln = {cv}") logger.info(f"pressure = {p}") @@ -220,7 +220,8 @@ def test_shock_init(ctx_factory): eos = IdealSingleGas() p = eos.pressure(initsoln) - assert discr.norm(actx.np.where(nodes_x < 0.5, p-xpl, p-xpr), np.inf) < tol + assert actx.to_numpy( + discr.norm(actx.np.where(nodes_x < 0.5, p-xpl, p-xpr), np.inf)) < tol @pytest.mark.parametrize("dim", [1, 2, 3]) @@ -254,15 +255,18 @@ def test_uniform(ctx_factory, dim): initsoln = initr(time=0.0, x_vec=nodes) tol = 1e-15 - assert discr.norm(initsoln.mass - 1.0, np.inf) < tol - assert discr.norm(initsoln.energy - 2.5, np.inf) < tol + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + assert inf_norm(initsoln.mass - 1.0) < tol + assert inf_norm(initsoln.energy - 2.5) < tol print(f"Uniform Soln:{initsoln}") eos = IdealSingleGas() p = eos.pressure(initsoln) print(f"Press:{p}") - assert discr.norm(p - 1.0, np.inf) < tol + assert inf_norm(p - 1.0) < tol @pytest.mark.parametrize("dim", [1, 2, 3]) @@ -301,30 +305,33 @@ def test_pulse(ctx_factory, dim): pulse = make_pulse(amp=amp, r0=r0, w=w, r=nodes) print(f"Pulse = {pulse}") + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + # does it return the expected exponential? pulse_check = actx.np.exp(-.5 * r2) print(f"exact: {pulse_check}") pulse_resid = pulse - pulse_check print(f"pulse residual: {pulse_resid}") - assert(discr.norm(pulse_resid, np.inf) < tol) + assert(inf_norm(pulse_resid) < tol) # proper scaling with amplitude? amp = 2.0 pulse = 0 pulse = make_pulse(amp=amp, r0=r0, w=w, r=nodes) pulse_resid = pulse - (pulse_check + pulse_check) - assert(discr.norm(pulse_resid, np.inf) < tol) + assert(inf_norm(pulse_resid) < tol) # proper scaling with r? amp = 1.0 rcheck = np.sqrt(2.0) * nodes pulse = make_pulse(amp=amp, r0=r0, w=w, r=rcheck) - assert(discr.norm(pulse - (pulse_check * pulse_check), np.inf) < tol) + assert(inf_norm(pulse - (pulse_check * pulse_check)) < tol) # proper scaling with w? w = w / np.sqrt(2.0) pulse = make_pulse(amp=amp, r0=r0, w=w, r=nodes) - assert(discr.norm(pulse - (pulse_check * pulse_check), np.inf) < tol) + assert(inf_norm(pulse - (pulse_check * pulse_check)) < tol) @pytest.mark.parametrize("dim", [1, 2, 3]) @@ -370,12 +377,15 @@ def test_multilump(ctx_factory, dim): numcvspec = len(cv.species_mass) print(f"get_num_species = {numcvspec}") + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + assert numcvspec == nspecies - assert discr.norm(cv.mass - rho0) == 0.0 + assert inf_norm(cv.mass - rho0) == 0.0 p = 0.4 * (cv.energy - 0.5 * np.dot(cv.momentum, cv.momentum) / cv.mass) exp_p = 1.0 - errmax = discr.norm(p - exp_p, np.inf) + errmax = inf_norm(p - exp_p) species_mass = cv.species_mass spec_r = make_obj_array([nodes - centers[i] for i in range(nspecies)]) @@ -389,7 +399,7 @@ def test_multilump(ctx_factory, dim): print(f"exp_mass = {exp_mass}") print(f"mass_resid = {mass_resid}") - assert discr.norm(mass_resid, np.inf) == 0.0 + assert inf_norm(mass_resid) == 0.0 logger.info(f"lump_soln = {cv}") logger.info(f"pressure = {p}") diff --git a/test/test_inviscid.py b/test/test_inviscid.py index 37f0cea94..c97b37501 100644 --- a/test/test_inviscid.py +++ b/test/test_inviscid.py @@ -175,16 +175,20 @@ def test_inviscid_flux_components(actx_factory, dim): cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) p = eos.pressure(cv) flux = inviscid_flux(discr, eos, cv) - assert discr.norm(p - p_exact, np.inf) < tolerance + + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + assert inf_norm(p - p_exact) < tolerance logger.info(f"{dim}d flux = {flux}") # for velocity zero, these components should be == zero - assert discr.norm(flux.mass, 2) == 0.0 - assert discr.norm(flux.energy, 2) == 0.0 + assert inf_norm(flux.mass) == 0.0 + assert inf_norm(flux.energy) == 0.0 # The momentum diagonal should be p # Off-diagonal should be identically 0 - assert discr.norm(flux.momentum - p0*np.identity(dim), np.inf) < tolerance + assert inf_norm(flux.momentum - p0*np.identity(dim)) < tolerance @pytest.mark.parametrize(("dim", "livedim"), [ @@ -230,19 +234,23 @@ def test_inviscid_mom_flux_components(actx_factory, dim, livedim): ) cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) p = eos.pressure(cv) - assert discr.norm(p - p_exact, np.inf) < tolerance + + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + assert inf_norm(p - p_exact) < tolerance flux = inviscid_flux(discr, eos, cv) logger.info(f"{dim}d flux = {flux}") vel_exact = mom / mass # first two components should be nonzero in livedim only - assert discr.norm(flux.mass - mom, np.inf) == 0 + assert inf_norm(flux.mass - mom) == 0 eflux_exact = (energy + p_exact)*vel_exact - assert discr.norm(flux.energy - eflux_exact, np.inf) == 0 + assert inf_norm(flux.energy - eflux_exact) == 0 logger.info("Testing momentum") xpmomflux = mass*np.outer(vel_exact, vel_exact) + p_exact*np.identity(dim) - assert discr.norm(flux.momentum - xpmomflux, np.inf) < tolerance + assert inf_norm(flux.momentum - xpmomflux) < tolerance @pytest.mark.parametrize("nspecies", [0, 10]) @@ -302,7 +310,7 @@ def test_facial_flux(actx_factory, nspecies, order, dim): def inf_norm(data): if len(data) > 0: - return discr.norm(data, np.inf, dd="all_faces") + return actx.to_numpy(discr.norm(data, np.inf, dd="all_faces")) else: return 0.0 diff --git a/test/test_lazy.py b/test/test_lazy.py index e4af85344..75e645e0f 100644 --- a/test/test_lazy.py +++ b/test/test_lazy.py @@ -79,7 +79,9 @@ def componentwise_norm(a): from mirgecom.fluid import ConservedVars if isinstance(a, ConservedVars): return componentwise_norm(a.join()) - return obj_array_vectorize(lambda b: discr.norm(b, np.inf), a) + from arraycontext import get_container_context_recursively + actx = get_container_context_recursively(a) + return obj_array_vectorize(lambda b: actx.to_numpy(discr.norm(b, np.inf)), a) lhs = componentwise_norm(x - y) rhs = np.maximum( diff --git a/test/test_operators.py b/test/test_operators.py index 049a67b31..d60a8ec80 100644 --- a/test/test_operators.py +++ b/test/test_operators.py @@ -217,10 +217,13 @@ def sym_eval(expr, x_vec): test_data = test_func(nodes) exact_grad = grad_test_func(nodes) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + if isinstance(test_data, ConservedVars): - err_scale = discr.norm(exact_grad.join(), np.inf) + err_scale = inf_norm(exact_grad.join()) else: - err_scale = discr.norm(exact_grad, np.inf) + err_scale = inf_norm(exact_grad) if err_scale <= 1e-16: err_scale = 1 @@ -238,13 +241,13 @@ def sym_eval(expr, x_vec): dim=dim, q=grad_operator(discr, test_data.join(), test_data_flux_bnd.join()) ) - grad_err = discr.norm((test_grad - exact_grad).join(), np.inf)/err_scale + grad_err = inf_norm((test_grad - exact_grad).join())/err_scale else: test_grad = grad_operator(discr, test_data, test_data_flux_bnd) - grad_err = discr.norm(test_grad - exact_grad, np.inf)/err_scale + grad_err = inf_norm(test_grad - exact_grad)/err_scale print(f"{test_grad=}") - eoc.add_data_point(h_max, grad_err) + eoc.add_data_point(actx.to_numpy(h_max), grad_err) assert ( eoc.order_estimate() >= order - 0.5 diff --git a/test/test_restart.py b/test/test_restart.py index 287598e01..2b6b6dcdb 100644 --- a/test/test_restart.py +++ b/test/test_restart.py @@ -76,4 +76,4 @@ def test_restart_cv(actx_factory, nspecies): restart_data = read_restart_data(actx, rst_filename) resid = test_state - restart_data["state"] - assert discr.norm(resid.join(), np.inf) == 0 + assert actx.to_numpy(discr.norm(resid.join(), np.inf)) == 0 diff --git a/test/test_simutil.py b/test/test_simutil.py index 9f78062db..a590abcd9 100644 --- a/test/test_simutil.py +++ b/test/test_simutil.py @@ -134,7 +134,7 @@ def test_analytic_comparison(actx_factory): cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) resid = vortex_soln - cv - expected_errors = [discr.norm(v, np.inf) for v in resid.join()] + expected_errors = [actx.to_numpy(discr.norm(v, np.inf)) for v in resid.join()] errors = compare_fluid_solutions(discr, cv, cv) assert max(errors) == 0 diff --git a/test/test_symbolic.py b/test/test_symbolic.py index c2337f1e6..489476014 100644 --- a/test/test_symbolic.py +++ b/test/test_symbolic.py @@ -164,7 +164,7 @@ def test_symbolic_evaluation(actx_factory): expected_f = np.exp(-t) * actx.np.cos(nodes[0]) * actx.np.sin(nodes[1]) - assert discr.norm(f - expected_f)/discr.norm(expected_f) < 1e-12 + assert actx.to_numpy(discr.norm(f - expected_f)/discr.norm(expected_f)) < 1e-12 if __name__ == "__main__": diff --git a/test/test_viscous.py b/test/test_viscous.py index 3fffb39e0..d30e37cdf 100644 --- a/test/test_viscous.py +++ b/test/test_viscous.py @@ -105,7 +105,7 @@ def test_viscous_stress_tensor(actx_factory, transport_model): tau = viscous_stress_tensor(discr, eos, cv, grad_cv) # The errors come from grad_v - assert discr.norm(tau - exp_tau, np.inf) < 1e-12 + assert actx.to_numpy(discr.norm(tau - exp_tau, np.inf)) < 1e-12 # Box grid generator widget lifted from @majosm and slightly bent @@ -189,6 +189,9 @@ def cv_flux_boundary(btag): discr = EagerDGDiscretization(actx, mesh, order=order) nodes = thaw(actx, discr.nodes()) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + # compute max element size from grudge.dt_utils import h_max_from_volume h_max = h_max_from_volume(discr) @@ -208,10 +211,10 @@ def cv_flux_boundary(btag): xp_tau = mu * (xp_grad_v + xp_grad_v.transpose()) # sanity check the gradient: - relerr_scale_e = 1.0 / discr.norm(xp_grad_cv.energy, np.inf) - relerr_scale_p = 1.0 / discr.norm(xp_grad_cv.momentum, np.inf) - graderr_e = discr.norm((grad_cv.energy - xp_grad_cv.energy), np.inf) - graderr_p = discr.norm((grad_cv.momentum - xp_grad_cv.momentum), np.inf) + relerr_scale_e = 1.0 / inf_norm(xp_grad_cv.energy) + relerr_scale_p = 1.0 / inf_norm(xp_grad_cv.momentum) + graderr_e = inf_norm(grad_cv.energy - xp_grad_cv.energy) + graderr_p = inf_norm(grad_cv.momentum - xp_grad_cv.momentum) graderr_e *= relerr_scale_e graderr_p *= relerr_scale_p assert graderr_e < 5e-7 @@ -231,14 +234,14 @@ def cv_flux_boundary(btag): grad_t = op.local_grad(discr, temperature) # sanity check - assert discr.norm(grad_p - xp_grad_p, np.inf)*dpscal < 5e-9 - assert discr.norm(grad_t - xp_grad_t, np.inf)*tscal < 5e-9 + assert inf_norm(grad_p - xp_grad_p)*dpscal < 5e-9 + assert inf_norm(grad_t - xp_grad_t)*tscal < 5e-9 # verify heat flux from mirgecom.viscous import conductive_heat_flux heat_flux = conductive_heat_flux(discr, eos, cv, grad_t) xp_heat_flux = -kappa*xp_grad_t - assert discr.norm(heat_flux - xp_heat_flux, np.inf) < 2e-8 + assert inf_norm(heat_flux - xp_heat_flux) < 2e-8 # verify diffusive mass flux is zilch (no scalar components) from mirgecom.viscous import diffusive_flux @@ -251,17 +254,17 @@ def cv_flux_boundary(btag): vflux = viscous_flux(discr, eos, cv, grad_cv, grad_t) efluxerr = ( - discr.norm(vflux.energy - xp_e_flux, np.inf) - / discr.norm(xp_e_flux, np.inf) + inf_norm(vflux.energy - xp_e_flux) + / inf_norm(xp_e_flux) ) momfluxerr = ( - discr.norm(vflux.momentum - xp_mom_flux, np.inf) - / discr.norm(xp_mom_flux, np.inf) + inf_norm(vflux.momentum - xp_mom_flux) + / inf_norm(xp_mom_flux) ) - assert discr.norm(vflux.mass, np.inf) == 0 - e_eoc_rec.add_data_point(h_max, efluxerr) - p_eoc_rec.add_data_point(h_max, momfluxerr) + assert inf_norm(vflux.mass) == 0 + e_eoc_rec.add_data_point(actx.to_numpy(h_max), efluxerr) + p_eoc_rec.add_data_point(actx.to_numpy(h_max), momfluxerr) assert ( e_eoc_rec.order_estimate() >= order - 0.5 @@ -333,15 +336,18 @@ def test_species_diffusive_flux(actx_factory): from mirgecom.viscous import diffusive_flux j = diffusive_flux(discr, eos, cv, grad_cv) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + tol = 1e-10 for idim in range(dim): ispec = 2*idim exact_dy = np.array([((ispec+1)*(idim*dim+1))*(iidim+1) for iidim in range(dim)]) exact_j = -massval * d_alpha[ispec] * exact_dy - assert discr.norm(j[ispec] - exact_j, np.inf) < tol + assert inf_norm(j[ispec] - exact_j) < tol exact_j = massval * d_alpha[ispec+1] * exact_dy - assert discr.norm(j[ispec+1] - exact_j, np.inf) < tol + assert inf_norm(j[ispec+1] - exact_j) < tol def test_diffusive_heat_flux(actx_factory): @@ -403,15 +409,18 @@ def test_diffusive_heat_flux(actx_factory): from mirgecom.viscous import diffusive_flux j = diffusive_flux(discr, eos, cv, grad_cv) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + tol = 1e-10 for idim in range(dim): ispec = 2*idim exact_dy = np.array([((ispec+1)*(idim*dim+1))*(iidim+1) for iidim in range(dim)]) exact_j = -massval * d_alpha[ispec] * exact_dy - assert discr.norm(j[ispec] - exact_j, np.inf) < tol + assert inf_norm(j[ispec] - exact_j) < tol exact_j = massval * d_alpha[ispec+1] * exact_dy - assert discr.norm(j[ispec+1] - exact_j, np.inf) < tol + assert inf_norm(j[ispec+1] - exact_j) < tol @pytest.mark.parametrize("array_valued", [False, True]) @@ -462,7 +471,7 @@ def test_local_max_species_diffusivity(actx_factory, dim, array_valued): expected *= f calculated = get_local_max_species_diffusivity(actx, discr, d_alpha) - assert discr.norm(calculated-expected, np.inf) == 0 + assert actx.to_numpy(discr.norm(calculated-expected, np.inf)) == 0 @pytest.mark.parametrize("dim", [1, 2, 3]) @@ -519,4 +528,4 @@ def test_viscous_timestep(actx_factory, dim, mu, vel): dt_expected = chlen / (speed_total + (mu / chlen)) error = (dt_expected - dt_field) / dt_expected - assert discr.norm(error, np.inf) == 0 + assert actx.to_numpy(discr.norm(error, np.inf)) == 0 diff --git a/test/test_wave.py b/test/test_wave.py index d33c98259..5e4cf8d3b 100644 --- a/test/test_wave.py +++ b/test/test_wave.py @@ -184,7 +184,7 @@ def sym_eval(expr, t): expected_rhs = sym_eval(sym_rhs, t_check) - rel_linf_err = ( + rel_linf_err = actx.to_numpy( discr.norm(rhs - expected_rhs, np.inf) / discr.norm(expected_rhs, np.inf)) eoc_rec.add_data_point(1./n, rel_linf_err) @@ -269,8 +269,11 @@ def get_rhs(t, w): ("v_expected", expected_fields[1:]), ]) - err = discr.norm(fields-expected_fields, np.inf) - max_err = discr.norm(expected_fields, np.inf) + def inf_norm(x): + return actx.to_numpy(discr.norm(x, np.inf)) + + err = inf_norm(fields-expected_fields) + max_err = inf_norm(expected_fields) assert err < max_err From dbf1aea654fd33ae2f092d067550dc50db61bd6d Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 29 Oct 2021 10:33:47 -0500 Subject: [PATCH 345/385] Install mirgecom from existing source, not from scratch. --- .ci-support/production-install.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh index 00b7f228c..f74406974 100644 --- a/.ci-support/production-install.sh +++ b/.ci-support/production-install.sh @@ -48,9 +48,11 @@ echo "PRODUCTION_FORK=$PRODUCTION_FORK" echo "PRODUCTION_BRANCH=$PRODUCTION_BRANCH" # Install the production branch with emirge -./install.sh --fork=${DEVELOPMENT_FORK} --branch=${DEVELOPMENT_BRANCH} +# Try something less fragile +# ./install.sh --fork=${DEVELOPMENT_FORK} --branch=${DEVELOPMENT_BRANCH} +./install.sh --skip-clone --conda-env=../mirgecom/conda_env.yml --pip-pkgs=../mirgecom/requirements.txt source config/activate_env.sh -cd mirgecom +cd ../mirgecom # This junk is needed to be able to execute git commands properly git config user.email "ci-runner@ci.machine.com" From 24340fa9220aa35a6c1ba4c296f1cc53b6cfb4e5 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 29 Oct 2021 10:38:21 -0500 Subject: [PATCH 346/385] Spell conda-env.yml correctly --- .ci-support/production-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh index f74406974..635e30874 100644 --- a/.ci-support/production-install.sh +++ b/.ci-support/production-install.sh @@ -50,7 +50,7 @@ echo "PRODUCTION_BRANCH=$PRODUCTION_BRANCH" # Install the production branch with emirge # Try something less fragile # ./install.sh --fork=${DEVELOPMENT_FORK} --branch=${DEVELOPMENT_BRANCH} -./install.sh --skip-clone --conda-env=../mirgecom/conda_env.yml --pip-pkgs=../mirgecom/requirements.txt +./install.sh --skip-clone --conda-env=../mirgecom/conda-env.yml --pip-pkgs=../mirgecom/requirements.txt source config/activate_env.sh cd ../mirgecom From 044c852c768234366f2dc8c0631d384902b77a99 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 29 Oct 2021 10:57:06 -0500 Subject: [PATCH 347/385] Set an install prefix --- .ci-support/production-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh index 635e30874..7fc3f8745 100644 --- a/.ci-support/production-install.sh +++ b/.ci-support/production-install.sh @@ -50,7 +50,7 @@ echo "PRODUCTION_BRANCH=$PRODUCTION_BRANCH" # Install the production branch with emirge # Try something less fragile # ./install.sh --fork=${DEVELOPMENT_FORK} --branch=${DEVELOPMENT_BRANCH} -./install.sh --skip-clone --conda-env=../mirgecom/conda-env.yml --pip-pkgs=../mirgecom/requirements.txt +./install.sh --skip-clone --install-prefix=../ --conda-env=../mirgecom/conda-env.yml --pip-pkgs=../mirgecom/requirements.txt source config/activate_env.sh cd ../mirgecom From 12e1360c5a773bd4ecad7411609edd84512f34f2 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 29 Oct 2021 11:08:53 -0500 Subject: [PATCH 348/385] Find activation script in install location. --- .ci-support/production-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh index 7fc3f8745..8260ec315 100644 --- a/.ci-support/production-install.sh +++ b/.ci-support/production-install.sh @@ -51,7 +51,7 @@ echo "PRODUCTION_BRANCH=$PRODUCTION_BRANCH" # Try something less fragile # ./install.sh --fork=${DEVELOPMENT_FORK} --branch=${DEVELOPMENT_BRANCH} ./install.sh --skip-clone --install-prefix=../ --conda-env=../mirgecom/conda-env.yml --pip-pkgs=../mirgecom/requirements.txt -source config/activate_env.sh +source ../config/activate_env.sh cd ../mirgecom # This junk is needed to be able to execute git commands properly From 4151991cf5d8b57bd808ec208b4e621ba840e21e Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 12:27:13 -0500 Subject: [PATCH 349/385] Configure production testing --- .ci-support/production-testing-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 6cf17aa41..5517dc273 100644 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -16,7 +16,7 @@ set -x # patched by the incoming development. The following vars control the # production environment: # -# export PRODUCTION_BRANCH="" # The base production branch to be installed by emirge +export PRODUCTION_BRANCH="ci-test-production" # The base production branch to be installed by emirge # export PRODUCTION_FORK="" # The fork/home of production changes (if any) # # Multiple production drivers are supported. The user should provide a ':'-delimited From 9d6e82acaf30a6bad5482d76586d7034b0ca91ca Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 12:56:19 -0500 Subject: [PATCH 350/385] Debug ci more --- .ci-support/production-install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh index 8260ec315..a37b48730 100644 --- a/.ci-support/production-install.sh +++ b/.ci-support/production-install.sh @@ -53,12 +53,13 @@ echo "PRODUCTION_BRANCH=$PRODUCTION_BRANCH" ./install.sh --skip-clone --install-prefix=../ --conda-env=../mirgecom/conda-env.yml --pip-pkgs=../mirgecom/requirements.txt source ../config/activate_env.sh cd ../mirgecom - +pwd # This junk is needed to be able to execute git commands properly git config user.email "ci-runner@ci.machine.com" git config user.name "CI Runner" # Merge in the production environment +git status git remote add production https://github.com/${PRODUCTION_FORK}/mirgecom git fetch production git merge production/${PRODUCTION_BRANCH} --no-edit From ee2d0994a450e2773931930f4a87bf8b524263e3 Mon Sep 17 00:00:00 2001 From: Matthias Diener Date: Fri, 29 Oct 2021 13:16:31 -0500 Subject: [PATCH 351/385] full clone --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index db21db82d..2c2c26945 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -164,6 +164,8 @@ jobs: steps: - uses: actions/checkout@v2 + with: + fetch-depth: '0' - name: Prepare production environment run: | [[ $(uname) == Linux ]] && sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev From 7c0fc6febf8ede1b1e671922078a91173413979c Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 14:06:28 -0500 Subject: [PATCH 352/385] Clean up production testing scripts --- .ci-support/production-driver-install.sh | 16 ++++----- .ci-support/production-install.sh | 43 ++++++++++++------------ .github/workflows/ci.yaml | 14 +++++--- 3 files changed, 38 insertions(+), 35 deletions(-) diff --git a/.ci-support/production-driver-install.sh b/.ci-support/production-driver-install.sh index e01c71aca..de040690d 100644 --- a/.ci-support/production-driver-install.sh +++ b/.ci-support/production-driver-install.sh @@ -15,14 +15,14 @@ # set -x -DEVELOPMENT_BRANCH="$GITHUB_HEAD_REF" # this will be empty for main -PRODUCTION_DRIVERS="" -if [ -n "$DEVELOPMENT_BRANCH" ]; then - PRODUCTION_ENV_FILE="$1" - if [ -e "$PRODUCTION_ENV_FILE" ]; then - . $PRODUCTION_ENV_FILE - fi -fi +# DEVELOPMENT_BRANCH="$GITHUB_HEAD_REF" # this will be empty for main +# PRODUCTION_DRIVERS="" +# if [ -n "$DEVELOPMENT_BRANCH" ]; then +# PRODUCTION_ENV_FILE="$1" +# if [ -e "$PRODUCTION_ENV_FILE" ]; then +# . $PRODUCTION_ENV_FILE +# fi +# fi # Set to default if testing main, or user left it empty PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@y2-production:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar"} OIFS="$IFS" diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh index a37b48730..e15291112 100644 --- a/.ci-support/production-install.sh +++ b/.ci-support/production-install.sh @@ -22,37 +22,35 @@ # set -x -if [ -n "$DEVELOPMENT_BRANCH" ]; then - PRODUCTION_ENV_FILE="$1" - if [ -e "$PRODUCTION_ENV_FILE" ]; then - echo "Reading production configuration for ${DEVELOPMENT_BRANCH}." - . $PRODUCTION_ENV_FILE - else - echo "Using default production configuration for ${DEVELOPMENT_BRANCH}." - echo "To customize, set up .ci-support/production-testing-env.sh." - fi -fi -DEVELOPMENT_BRANCH=${DEVELOPMENT_BRANCH:-"main"} -DEVELOPMENT_FORK=${DEVELOPMENT_FORK:-"illinois-ceesd"} +EMIRGE_INSTALL_PATH=${1:-"."} +#if [ -n "$DEVELOPMENT_BRANCH" ]; then +# PRODUCTION_ENV_FILE="$1" +# if [ -e "$PRODUCTION_ENV_FILE" ]; then +# echo "Reading production configuration for ${DEVELOPMENT_BRANCH}." +# . $PRODUCTION_ENV_FILE +# else +# echo "Using default production configuration for ${DEVELOPMENT_BRANCH}." +# echo "To customize, set up .ci-support/production-testing-env.sh." +# fi +#fi +# DEVELOPMENT_BRANCH="$GITHUB_HEAD_REF" # this will be empty for main +# DEVELOPMENT_BRANCH=${DEVELOPMENT_BRANCH:-"main"} +# DEVELOPMENT_FORK=${DEVELOPMENT_FORK:-"illinois-ceesd"} PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"y1-production"} PRODUCTION_FORK=${PRODUCTION_FORK:-"illinois-ceesd"} -echo "Production environment settings:" -if [ -n "${PRODUCTION_ENV_FILE}" ]; then - echo "PRODUCTION_ENV_FILE=$PRODUCTION_ENV_FILE" - cat ${PRODUCTION_ENV_FILE} -fi -echo "DEVELOPMENT_FORK=$DEVELOPMENT_FORK" -echo "DEVELOPMENT_BRANCH=$DEVELOPMENT_BRANCH" +echo "EMIRGE_INSTALL_PATH=${EMIRGE_INSTALL_PATH}" +# echo "DEVELOPMENT_FORK=$DEVELOPMENT_FORK" +# echo "DEVELOPMENT_BRANCH=$DEVELOPMENT_BRANCH" echo "PRODUCTION_FORK=$PRODUCTION_FORK" echo "PRODUCTION_BRANCH=$PRODUCTION_BRANCH" # Install the production branch with emirge # Try something less fragile # ./install.sh --fork=${DEVELOPMENT_FORK} --branch=${DEVELOPMENT_BRANCH} -./install.sh --skip-clone --install-prefix=../ --conda-env=../mirgecom/conda-env.yml --pip-pkgs=../mirgecom/requirements.txt -source ../config/activate_env.sh -cd ../mirgecom +./install.sh --skip-clone --install-prefix=${EMIRGE_INSTALL_PATH}/ --conda-env=${EMIRGE_INSTALL_PATH}/mirgecom/conda-env.yml --pip-pkgs=${EMIRGE_INSTALL_PATH}/mirgecom/requirements.txt +source ${EMIRGE_INSTALL_PATH}/config/activate_env.sh +cd ${EMIRGE_INSTALL_PATH}/mirgecom pwd # This junk is needed to be able to execute git commands properly git config user.email "ci-runner@ci.machine.com" @@ -64,3 +62,4 @@ git remote add production https://github.com/${PRODUCTION_FORK}/mirgecom git fetch production git merge production/${PRODUCTION_BRANCH} --no-edit pip install -r requirements.txt +cd - diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2c2c26945..d19e986fc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -170,16 +170,20 @@ jobs: run: | [[ $(uname) == Linux ]] && sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev [[ $(uname) == Darwin ]] && brew update && brew install mpich + MIRGEDIR=$(pwd) + cat .ci-support/production-testing-env.sh + . .ci-support/production-testing-env.sh cd .. git clone https://github.com/illinois-ceesd/emirge emirge.y1 cd emirge.y1 - . ../mirgecom/.ci-support/production-testing-env.sh - . ../mirgecom/.ci-support/production-install.sh ../mirgecom/.ci-support/production-testing-env.sh + . ../mirgecom/.ci-support/production-install.sh $(MIRGEDIR)/.. - name: Run production test run: | - cd .. - source emirge.y1/config/activate_env.sh - . mirgecom/.ci-support/production-driver-install.sh mirgecom/.ci-support/production-testing-env.sh + MIRGEDIR=$(pwd) + . .ci-support/production-testing-env.sh + cd ../ + source ${MIRGEDIR}/../config/activate_env.sh + . mirgecom/.ci-support/production-driver-install.sh for production_driver in $(ls | grep "production_driver_"); do cd "$production_driver"/smoke_test From 65b8da233af6381980f9f024c7766fc4f9fe2e55 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 14:11:26 -0500 Subject: [PATCH 353/385] Fix script bug --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d19e986fc..dda8f3173 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -176,7 +176,7 @@ jobs: cd .. git clone https://github.com/illinois-ceesd/emirge emirge.y1 cd emirge.y1 - . ../mirgecom/.ci-support/production-install.sh $(MIRGEDIR)/.. + . ../mirgecom/.ci-support/production-install.sh ${MIRGEDIR}/.. - name: Run production test run: | MIRGEDIR=$(pwd) From 4142d8357ca7867adf48d17d1ea90e57be39ba47 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 16:38:55 -0500 Subject: [PATCH 354/385] Clean up further --- .ci-support/production-driver-install.sh | 17 ++++---------- .ci-support/production-install.sh | 29 +++++++----------------- 2 files changed, 12 insertions(+), 34 deletions(-) diff --git a/.ci-support/production-driver-install.sh b/.ci-support/production-driver-install.sh index de040690d..fb7a6d0d9 100644 --- a/.ci-support/production-driver-install.sh +++ b/.ci-support/production-driver-install.sh @@ -6,25 +6,16 @@ # what production test is installed, the env setup script should set # the following: # +# # PRODUCTION_DRIVERS = ':' delimited list "fork/repo@branch" # (See the example default value below) # # The default values result in an install of the Y1 nozzle driver and # Wyatt Hagen's isolator driver that work with current MIRGE-Com # production branch: mirgecom@y1-production. -# -set -x - -# DEVELOPMENT_BRANCH="$GITHUB_HEAD_REF" # this will be empty for main -# PRODUCTION_DRIVERS="" -# if [ -n "$DEVELOPMENT_BRANCH" ]; then -# PRODUCTION_ENV_FILE="$1" -# if [ -e "$PRODUCTION_ENV_FILE" ]; then -# . $PRODUCTION_ENV_FILE -# fi -# fi -# Set to default if testing main, or user left it empty PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@y2-production:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar"} +# Loop over the production drivers, clone them, and prepare for execution +set -x OIFS="$IFS" IFS=':'; for production_driver_string in $PRODUCTION_DRIVERS; do @@ -34,7 +25,7 @@ do PRODUCTION_DRIVER_DIR="production_driver_$PRODUCTION_DRIVER_NAME" git clone -b "$PRODUCTION_DRIVER_BRANCH" https\://github.com/"$PRODUCTION_DRIVER_REPO" "$PRODUCTION_DRIVER_DIR" cd "$PRODUCTION_DRIVER_DIR"/smoke_test - ln -s *.py driver.py + ln -s *.py driver.py # name the driver generically cd ../.. done IFS="$OIFS" diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh index e15291112..a681268f2 100644 --- a/.ci-support/production-install.sh +++ b/.ci-support/production-install.sh @@ -23,43 +23,30 @@ set -x EMIRGE_INSTALL_PATH=${1:-"."} -#if [ -n "$DEVELOPMENT_BRANCH" ]; then -# PRODUCTION_ENV_FILE="$1" -# if [ -e "$PRODUCTION_ENV_FILE" ]; then -# echo "Reading production configuration for ${DEVELOPMENT_BRANCH}." -# . $PRODUCTION_ENV_FILE -# else -# echo "Using default production configuration for ${DEVELOPMENT_BRANCH}." -# echo "To customize, set up .ci-support/production-testing-env.sh." -# fi -#fi -# DEVELOPMENT_BRANCH="$GITHUB_HEAD_REF" # this will be empty for main -# DEVELOPMENT_BRANCH=${DEVELOPMENT_BRANCH:-"main"} -# DEVELOPMENT_FORK=${DEVELOPMENT_FORK:-"illinois-ceesd"} PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"y1-production"} PRODUCTION_FORK=${PRODUCTION_FORK:-"illinois-ceesd"} echo "EMIRGE_INSTALL_PATH=${EMIRGE_INSTALL_PATH}" -# echo "DEVELOPMENT_FORK=$DEVELOPMENT_FORK" -# echo "DEVELOPMENT_BRANCH=$DEVELOPMENT_BRANCH" echo "PRODUCTION_FORK=$PRODUCTION_FORK" echo "PRODUCTION_BRANCH=$PRODUCTION_BRANCH" -# Install the production branch with emirge -# Try something less fragile -# ./install.sh --fork=${DEVELOPMENT_FORK} --branch=${DEVELOPMENT_BRANCH} +# Install the version of mirgecom we wish to test from source ./install.sh --skip-clone --install-prefix=${EMIRGE_INSTALL_PATH}/ --conda-env=${EMIRGE_INSTALL_PATH}/mirgecom/conda-env.yml --pip-pkgs=${EMIRGE_INSTALL_PATH}/mirgecom/requirements.txt + +# Activate the environment (needed?) source ${EMIRGE_INSTALL_PATH}/config/activate_env.sh cd ${EMIRGE_INSTALL_PATH}/mirgecom -pwd + # This junk is needed to be able to execute git commands properly git config user.email "ci-runner@ci.machine.com" git config user.name "CI Runner" -# Merge in the production environment -git status +# Making a dedicated production remote adds production forks git remote add production https://github.com/${PRODUCTION_FORK}/mirgecom git fetch production + +# Merge the production branch for testing the production drivers git merge production/${PRODUCTION_BRANCH} --no-edit +# Pick up any requirements.txt pip install -r requirements.txt cd - From 8f40e1785060c86c5c74d5741494d59bf35163a8 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 19:20:01 -0500 Subject: [PATCH 355/385] Add some production testing docs --- doc/development/pullrequests.rst | 71 ++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/doc/development/pullrequests.rst b/doc/development/pullrequests.rst index a34ef9c03..ef2646c58 100644 --- a/doc/development/pullrequests.rst +++ b/doc/development/pullrequests.rst @@ -206,6 +206,77 @@ commit, you can use a git hook such as the following one (save this script as While highly recommended, hooks can sometimes be annoying. After setting up your hooks, you can use ``git --no-verify`` or equivalently ``git -n`` to run ``git`` commands without triggering the hooks. +Production Testing in CI +------------------------ + +The CI testing in mirgecom includes a set of "production" tests which help +detect when a proposed change in a PR breaks the CEESD prediction capability +toolchain. + +When PRs run afoul of the CI production tests, it indicates that if the PR +change set merges to main, then the "production" capability of mirgecom will +not function until the production capability and the change set are brought +into accordance. + +The production tests may be prepared and executed from anywhere by +hand-executing the production test scripts found in ``.ci-support/``. The +following is an example workflow adjacent to what CI itself does for executing +the production tests. + +1. Check out the PR development (and optionally make a production branch) + + .. code:: bash + + $ # For ceesd-local branches: + $ git clone -b branch-name git@github.com:/illinois-ceesd/mirgecom + $ # Or for developer fork: + $ git clone -b branch-name git@github.com:/fork-name/mirgecom + $ cd mirgecom # or loopy, meshmode, ... + $ git switch -c branch-name-production # Optional production branch + +2. Set up the production environment and capability + + .. code:: bash + + $ # Load the customizable production environment + $ . .ci-support/production-testing-env.sh + $ # Merge the production branch + $ . .ci-support/merge-install-production-branch . + + If Step 2 fails, i.e. if there are merge conflicts, then those must + be resolved. Push the merged result to CEESD or a fork, and indicate + that version in the PRODUCTION_FORK, and PRODUCTION_BRANCH env vars in + ``.ci-support/production-testing-env.sh``. + +3. Grab and run the production tests + + .. code:: bash + + $ # Load env if needed + $ . .ci-support/production-testing-env.sh + $ # Get the production tests + $ . .ci-support/production-driver-install.sh . + $ # Run the production tests + $ . .ci-support/production-test-run.sh . + + Step 3 will clone the production driver repos to the current directory, + with each driver in its own directory. If any of the drivers fail to + work with the current development, then they may be modified into working + condition and then pushed to a repo. Indicate the location of the working + drivers in the PRODUCTION_DRIVERS env var customization in + ``.ci-support/production-testing-env.sh``. + +If the PR development required production environment customization in order to +pass production tests, then care and coordination will be required to get these +changes merged into the main mirgecom development. PR reviewers will help +with this process. + + .. important:: + + Any production environment customizations must be backed out before + merging the PR development to main. Never merge a PR development with + production environment customizations in place. + Merging a pull request ---------------------- From 8a461fab003cb2963889dfe29a1d1449adac49e6 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 19:32:27 -0500 Subject: [PATCH 356/385] Sharpen slightly --- doc/development/pullrequests.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/doc/development/pullrequests.rst b/doc/development/pullrequests.rst index ef2646c58..7424abb7b 100644 --- a/doc/development/pullrequests.rst +++ b/doc/development/pullrequests.rst @@ -220,8 +220,10 @@ into accordance. The production tests may be prepared and executed from anywhere by hand-executing the production test scripts found in ``.ci-support/``. The -following is an example workflow adjacent to what CI itself does for executing -the production tests. +following is an example workflow adjacent to what CI itself does for +executing the production tests. In the following example, the PR development +is assumed to be in a mirgecom branch called ``branch-name`` and possibly in a +fork called ``fork-name``. 1. Check out the PR development (and optionally make a production branch) @@ -262,20 +264,22 @@ the production tests. Step 3 will clone the production driver repos to the current directory, with each driver in its own directory. If any of the drivers fail to work with the current development, then they may be modified into working - condition and then pushed to a repo. Indicate the location of the working + condition and then pushed to a repo/branch. Indicate the location of the working drivers in the PRODUCTION_DRIVERS env var customization in ``.ci-support/production-testing-env.sh``. -If the PR development required production environment customization in order to +If the PR development requires production environment customization in order to pass production tests, then care and coordination will be required to get these changes merged into the main mirgecom development. PR reviewers will help -with this process. +with this process. If the PR is accepted for merging to main, then mirgecom +developers will update the production capabilities to be compatible with +the changes before merging. .. important:: Any production environment customizations must be backed out before merging the PR development to main. Never merge a PR development with - production environment customizations in place. + production environment customizations in-place. Merging a pull request ---------------------- From ad8a04bb60369b0bd0223f1c0e017884e90c9d90 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 20:16:49 -0500 Subject: [PATCH 357/385] Sharpen more --- doc/development/pullrequests.rst | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/doc/development/pullrequests.rst b/doc/development/pullrequests.rst index 7424abb7b..e466c657f 100644 --- a/doc/development/pullrequests.rst +++ b/doc/development/pullrequests.rst @@ -211,22 +211,26 @@ Production Testing in CI The CI testing in mirgecom includes a set of "production" tests which help detect when a proposed change in a PR breaks the CEESD prediction capability -toolchain. +toolchain. Most developments and PRs do not require special considerations +for the production tests, but any production test failures will require +a bit of care to resolve. When PRs run afoul of the CI production tests, it indicates that if the PR change set merges to main, then the "production" capability of mirgecom will not function until the production capability and the change set are brought into accordance. -The production tests may be prepared and executed from anywhere by -hand-executing the production test scripts found in ``.ci-support/``. The -following is an example workflow adjacent to what CI itself does for -executing the production tests. In the following example, the PR development -is assumed to be in a mirgecom branch called ``branch-name`` and possibly in a -fork called ``fork-name``. +To resolve CI production test failures for a development in PR, it is often useful +to run the production tests manually. The production tests may be prepared and +executed from anywhere by hand-executing the production test scripts found in +``.ci-support/``. The following is an example workflow adjacent to what CI itself +does for executing the production tests. 1. Check out the PR development (and optionally make a production branch) + The PR development is assumed to be in a mirgecom branch called ``branch-name`` + and possibly in a fork called ``fork-name``. + .. code:: bash $ # For ceesd-local branches: @@ -268,6 +272,13 @@ fork called ``fork-name``. drivers in the PRODUCTION_DRIVERS env var customization in ``.ci-support/production-testing-env.sh``. +4. Update the PR to reflect the change in production environment (if any) + + Push the customized production ``.ci-support/production-testing-env.sh`` + settings to the PR development branch. Upon push, mirgecom CI will + try the production tests again, now with the customized environment. + + If the PR development requires production environment customization in order to pass production tests, then care and coordination will be required to get these changes merged into the main mirgecom development. PR reviewers will help From 57adc5212eac1a2ae91e23c9e7193df3886ed79f Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 20:58:11 -0500 Subject: [PATCH 358/385] Breakout functionality for hand-running. --- .ci-support/install-mirge-from-source.sh | 34 ++++++++++++++ .../merge-install-production-branch.sh | 47 +++++++++++++++++++ .ci-support/production-drivers-install.sh | 31 ++++++++++++ .ci-support/production-testing-env.sh | 0 .ci-support/production-tests-run.sh | 16 +++++++ .github/workflows/ci.yaml | 15 ++---- 6 files changed, 133 insertions(+), 10 deletions(-) create mode 100755 .ci-support/install-mirge-from-source.sh create mode 100755 .ci-support/merge-install-production-branch.sh create mode 100755 .ci-support/production-drivers-install.sh mode change 100644 => 100755 .ci-support/production-testing-env.sh create mode 100755 .ci-support/production-tests-run.sh diff --git a/.ci-support/install-mirge-from-source.sh b/.ci-support/install-mirge-from-source.sh new file mode 100755 index 000000000..61ee48b7d --- /dev/null +++ b/.ci-support/install-mirge-from-source.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -x +# +# This script is designed to patch the CEESD production capability against +# a proposed change to illinois-ceesd/mirgecom@main. The script reads the +# environment config file `.ci-support/production-testing-env.sh`, that +# should set up the expected control variables, which are as follows: +# +# The proposed changes to test may be in a fork, or a local branch. For +# forks, the environment config files should set: +# +# DEVELOPMENT_FORK = The development fork (default=illinois-ceesd) +# +# The production capability to test against may be specified outright, or +# patched by the incoming development. The following vars control the +# production environment: +# +# PRODUCTION_BRANCH = The production branch (default=y1-production) +# PRODUCTION_FORK = The production fork (default=illinois-ceesd) +# +# If the environment file does not exist, the current development is +# tested against `mirgecom@y1-production`. +# +EMIRGE_INSTALL_PATH=${1:-"."} +PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"y1-production"} +PRODUCTION_FORK=${PRODUCTION_FORK:-"illinois-ceesd"} + +echo "EMIRGE_INSTALL_PATH=${EMIRGE_INSTALL_PATH}" +echo "PRODUCTION_FORK=$PRODUCTION_FORK" +echo "PRODUCTION_BRANCH=$PRODUCTION_BRANCH" + +# Install the version of mirgecom we wish to test from source +./install.sh --skip-clone --install-prefix=${EMIRGE_INSTALL_PATH}/ --conda-env=${EMIRGE_INSTALL_PATH}/mirgecom/conda-env.yml --pip-pkgs=${EMIRGE_INSTALL_PATH}/mirgecom/requirements.txt + diff --git a/.ci-support/merge-install-production-branch.sh b/.ci-support/merge-install-production-branch.sh new file mode 100755 index 000000000..0ca6c2858 --- /dev/null +++ b/.ci-support/merge-install-production-branch.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -x +# +# This script is designed to patch the CEESD production capability against +# a proposed change to illinois-ceesd/mirgecom@main. The script reads the +# environment config file `.ci-support/production-testing-env.sh`, that +# should set up the expected control variables, which are as follows: +# +# The proposed changes to test may be in a fork, or a local branch. For +# forks, the environment config files should set: +# +# DEVELOPMENT_FORK = The development fork (default=illinois-ceesd) +# +# The production capability to test against may be specified outright, or +# patched by the incoming development. The following vars control the +# production environment: +# +# PRODUCTION_BRANCH = The production branch (default=y1-production) +# PRODUCTION_FORK = The production fork (default=illinois-ceesd) +# +# If the environment file does not exist, the current development is +# tested against `mirgecom@y1-production`. +# +MIRGE_HOME=${1:-"."} +PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"y1-production"} +PRODUCTION_FORK=${PRODUCTION_FORK:-"illinois-ceesd"} + +echo "MIRGE_HOME=${MIRGE_HOME}" +echo "PRODUCTION_FORK=$PRODUCTION_FORK" +echo "PRODUCTION_BRANCH=$PRODUCTION_BRANCH" + +cd ${MIRGE_HOME} +git status + +# This junk is needed to be able to execute git commands properly +git config user.email "ci-runner@ci.machine.com" +git config user.name "CI Runner" + +# Making a dedicated production remote adds production forks +git remote add production https://github.com/${PRODUCTION_FORK}/mirgecom +git fetch production + +# Merge the production branch for testing the production drivers +git merge production/${PRODUCTION_BRANCH} --no-edit +# Pick up any requirements.txt +pip install -r requirements.txt +cd - diff --git a/.ci-support/production-drivers-install.sh b/.ci-support/production-drivers-install.sh new file mode 100755 index 000000000..f04273d88 --- /dev/null +++ b/.ci-support/production-drivers-install.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# This script is designed to install the CEESD production test used to +# check that changes to main don't tear up the production capability. +# The script takes one argument, the production environment setup file, +# which is typically `.ci-support/production-env-setup.sh`. To specify +# what production test is installed, the env setup script should set +# the following: +# +# PRODUCTION_DRIVERS = ':' delimited list "fork/repo@branch" +# (See the example default value below) +# +# The default values result in an install of the Y1 nozzle driver and +# Wyatt Hagen's isolator driver that work with current MIRGE-Com +# production branch: mirgecom@y1-production. +PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@y2-production:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar"} +# Loop over the production drivers, clone them, and prepare for execution +set -x +OIFS="$IFS" +IFS=':'; for production_driver_string in $PRODUCTION_DRIVERS; +do + PRODUCTION_DRIVER_BRANCH=$(printf "$production_driver_string" | cut -d "@" -f 2) + PRODUCTION_DRIVER_REPO=$(printf "$production_driver_string" | cut -d "@" -f 1) + PRODUCTION_DRIVER_NAME=$(printf "$PRODUCTION_DRIVER_REPO" | cut -d "/" -f 2) + PRODUCTION_DRIVER_DIR="production_driver_$PRODUCTION_DRIVER_NAME" + git clone -b "$PRODUCTION_DRIVER_BRANCH" https\://github.com/"$PRODUCTION_DRIVER_REPO" "$PRODUCTION_DRIVER_DIR" + cd "$PRODUCTION_DRIVER_DIR"/smoke_test + ln -s *.py driver.py # name the driver generically + cd ../.. +done +IFS="$OIFS" +set +x diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh old mode 100644 new mode 100755 diff --git a/.ci-support/production-tests-run.sh b/.ci-support/production-tests-run.sh new file mode 100755 index 000000000..33d0fd89f --- /dev/null +++ b/.ci-support/production-tests-run.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -x +# This script is designed to run the CEESD "production" drivers after +# they have been prepared by an external helper script called +# production-drivers-install.sh. The drivers are each expected to be +# in a directory called "production_driver_*" and are expected to have +# a test driver in "production_driver_*/smoke_test/driver.py". +DRIVERS_HOME=${1:-"."} +cd ${DRIVERS_HOME} +for production_driver in $(ls | grep "production_driver_"); +do + cd "$production_driver"/smoke_test + python -m mpi4py ./driver.py -i run_params.yaml + cd ../../ +done +cd - diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index dda8f3173..06a6c36c5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -176,17 +176,12 @@ jobs: cd .. git clone https://github.com/illinois-ceesd/emirge emirge.y1 cd emirge.y1 - . ../mirgecom/.ci-support/production-install.sh ${MIRGEDIR}/.. + . ../mirgecom/.ci-support/install-mirge-from-source.sh ${MIRGEDIR}/.. + . ../mirgecom/.ci-support/merge-install-production-branch.sh ${MIRGEDIR} - name: Run production test run: | MIRGEDIR=$(pwd) . .ci-support/production-testing-env.sh - cd ../ - source ${MIRGEDIR}/../config/activate_env.sh - . mirgecom/.ci-support/production-driver-install.sh - for production_driver in $(ls | grep "production_driver_"); - do - cd "$production_driver"/smoke_test - python -m mpi4py ./driver.py -i run_params.yaml - cd ../../ - done + source ../config/activate_env.sh + . .ci-support/production-drivers-install.sh . + . .ci-support/production-drivers-run.sh . From 837abd42fa50f7a697aab7e43ff1e4d0732fa7e2 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 21:21:01 -0500 Subject: [PATCH 359/385] Update CI for changes to production test scripts. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 06a6c36c5..00b27a18d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -177,10 +177,10 @@ jobs: git clone https://github.com/illinois-ceesd/emirge emirge.y1 cd emirge.y1 . ../mirgecom/.ci-support/install-mirge-from-source.sh ${MIRGEDIR}/.. + source ${MIRGEDIR}/../config/activate_env.sh . ../mirgecom/.ci-support/merge-install-production-branch.sh ${MIRGEDIR} - name: Run production test run: | - MIRGEDIR=$(pwd) . .ci-support/production-testing-env.sh source ../config/activate_env.sh . .ci-support/production-drivers-install.sh . From 0f3ca635c9f952231cde8576e838f451aeb7b735 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 21:45:43 -0500 Subject: [PATCH 360/385] Rename for consistency --- .../{production-tests-run.sh => production-drivers-run.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .ci-support/{production-tests-run.sh => production-drivers-run.sh} (100%) diff --git a/.ci-support/production-tests-run.sh b/.ci-support/production-drivers-run.sh similarity index 100% rename from .ci-support/production-tests-run.sh rename to .ci-support/production-drivers-run.sh From e9fde3844aa594c5bb44baba3491b3e049328e1a Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 21:53:15 -0500 Subject: [PATCH 361/385] Correct some script names in doc. --- doc/development/pullrequests.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/development/pullrequests.rst b/doc/development/pullrequests.rst index e466c657f..c670d139f 100644 --- a/doc/development/pullrequests.rst +++ b/doc/development/pullrequests.rst @@ -247,7 +247,7 @@ does for executing the production tests. $ # Load the customizable production environment $ . .ci-support/production-testing-env.sh $ # Merge the production branch - $ . .ci-support/merge-install-production-branch . + $ . .ci-support/merge-install-production-branch.sh . If Step 2 fails, i.e. if there are merge conflicts, then those must be resolved. Push the merged result to CEESD or a fork, and indicate @@ -261,9 +261,9 @@ does for executing the production tests. $ # Load env if needed $ . .ci-support/production-testing-env.sh $ # Get the production tests - $ . .ci-support/production-driver-install.sh . + $ . .ci-support/production-drivers-install.sh . $ # Run the production tests - $ . .ci-support/production-test-run.sh . + $ . .ci-support/production-drivers-run.sh . Step 3 will clone the production driver repos to the current directory, with each driver in its own directory. If any of the drivers fail to From 816418ca9177bbcd8d89c245283e2f122cb9e9ca Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 22:31:21 -0500 Subject: [PATCH 362/385] Clean up more, remove cruft. --- .ci-support/install-mirge-from-source.sh | 25 +-------- .../merge-install-production-branch.sh | 12 +---- .ci-support/production-driver-install.sh | 32 ------------ .ci-support/production-install.sh | 52 ------------------- .ci-support/production-testing-env.sh | 16 ++---- 5 files changed, 8 insertions(+), 129 deletions(-) delete mode 100644 .ci-support/production-driver-install.sh delete mode 100644 .ci-support/production-install.sh diff --git a/.ci-support/install-mirge-from-source.sh b/.ci-support/install-mirge-from-source.sh index 61ee48b7d..bfa55dd08 100755 --- a/.ci-support/install-mirge-from-source.sh +++ b/.ci-support/install-mirge-from-source.sh @@ -1,33 +1,12 @@ #!/bin/bash set -x # -# This script is designed to patch the CEESD production capability against -# a proposed change to illinois-ceesd/mirgecom@main. The script reads the -# environment config file `.ci-support/production-testing-env.sh`, that -# should set up the expected control variables, which are as follows: -# -# The proposed changes to test may be in a fork, or a local branch. For -# forks, the environment config files should set: -# -# DEVELOPMENT_FORK = The development fork (default=illinois-ceesd) -# -# The production capability to test against may be specified outright, or -# patched by the incoming development. The following vars control the -# production environment: -# -# PRODUCTION_BRANCH = The production branch (default=y1-production) -# PRODUCTION_FORK = The production fork (default=illinois-ceesd) -# -# If the environment file does not exist, the current development is -# tested against `mirgecom@y1-production`. +# This script is intended to install mirgecom from an uninstalled +# mirgecom source gotten from a fresh clone. # EMIRGE_INSTALL_PATH=${1:-"."} -PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"y1-production"} -PRODUCTION_FORK=${PRODUCTION_FORK:-"illinois-ceesd"} echo "EMIRGE_INSTALL_PATH=${EMIRGE_INSTALL_PATH}" -echo "PRODUCTION_FORK=$PRODUCTION_FORK" -echo "PRODUCTION_BRANCH=$PRODUCTION_BRANCH" # Install the version of mirgecom we wish to test from source ./install.sh --skip-clone --install-prefix=${EMIRGE_INSTALL_PATH}/ --conda-env=${EMIRGE_INSTALL_PATH}/mirgecom/conda-env.yml --pip-pkgs=${EMIRGE_INSTALL_PATH}/mirgecom/requirements.txt diff --git a/.ci-support/merge-install-production-branch.sh b/.ci-support/merge-install-production-branch.sh index 0ca6c2858..3733c1a7d 100755 --- a/.ci-support/merge-install-production-branch.sh +++ b/.ci-support/merge-install-production-branch.sh @@ -1,15 +1,10 @@ #!/bin/bash set -x # -# This script is designed to patch the CEESD production capability against +# This script is designed to patch the CEESD production capability into # a proposed change to illinois-ceesd/mirgecom@main. The script reads the # environment config file `.ci-support/production-testing-env.sh`, that -# should set up the expected control variables, which are as follows: -# -# The proposed changes to test may be in a fork, or a local branch. For -# forks, the environment config files should set: -# -# DEVELOPMENT_FORK = The development fork (default=illinois-ceesd) +# should set up the expected control variables. # # The production capability to test against may be specified outright, or # patched by the incoming development. The following vars control the @@ -18,9 +13,6 @@ set -x # PRODUCTION_BRANCH = The production branch (default=y1-production) # PRODUCTION_FORK = The production fork (default=illinois-ceesd) # -# If the environment file does not exist, the current development is -# tested against `mirgecom@y1-production`. -# MIRGE_HOME=${1:-"."} PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"y1-production"} PRODUCTION_FORK=${PRODUCTION_FORK:-"illinois-ceesd"} diff --git a/.ci-support/production-driver-install.sh b/.ci-support/production-driver-install.sh deleted file mode 100644 index fb7a6d0d9..000000000 --- a/.ci-support/production-driver-install.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# This script is designed to install the CEESD production test used to -# check that changes to main don't tear up the production capability. -# The script takes one argument, the production environment setup file, -# which is typically `.ci-support/production-env-setup.sh`. To specify -# what production test is installed, the env setup script should set -# the following: -# -# -# PRODUCTION_DRIVERS = ':' delimited list "fork/repo@branch" -# (See the example default value below) -# -# The default values result in an install of the Y1 nozzle driver and -# Wyatt Hagen's isolator driver that work with current MIRGE-Com -# production branch: mirgecom@y1-production. -PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@y2-production:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar"} -# Loop over the production drivers, clone them, and prepare for execution -set -x -OIFS="$IFS" -IFS=':'; for production_driver_string in $PRODUCTION_DRIVERS; -do - PRODUCTION_DRIVER_BRANCH=$(printf "$production_driver_string" | cut -d "@" -f 2) - PRODUCTION_DRIVER_REPO=$(printf "$production_driver_string" | cut -d "@" -f 1) - PRODUCTION_DRIVER_NAME=$(printf "$PRODUCTION_DRIVER_REPO" | cut -d "/" -f 2) - PRODUCTION_DRIVER_DIR="production_driver_$PRODUCTION_DRIVER_NAME" - git clone -b "$PRODUCTION_DRIVER_BRANCH" https\://github.com/"$PRODUCTION_DRIVER_REPO" "$PRODUCTION_DRIVER_DIR" - cd "$PRODUCTION_DRIVER_DIR"/smoke_test - ln -s *.py driver.py # name the driver generically - cd ../.. -done -IFS="$OIFS" -set +x diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh deleted file mode 100644 index a681268f2..000000000 --- a/.ci-support/production-install.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# -# This script is designed to patch the CEESD production capability against -# a proposed change to illinois-ceesd/mirgecom@main. The script reads the -# environment config file `.ci-support/production-testing-env.sh`, that -# should set up the expected control variables, which are as follows: -# -# The proposed changes to test may be in a fork, or a local branch. For -# forks, the environment config files should set: -# -# DEVELOPMENT_FORK = The development fork (default=illinois-ceesd) -# -# The production capability to test against may be specified outright, or -# patched by the incoming development. The following vars control the -# production environment: -# -# PRODUCTION_BRANCH = The production branch (default=y1-production) -# PRODUCTION_FORK = The production fork (default=illinois-ceesd) -# -# If the environment file does not exist, the current development is -# tested against `mirgecom@y1-production`. -# -set -x - -EMIRGE_INSTALL_PATH=${1:-"."} -PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"y1-production"} -PRODUCTION_FORK=${PRODUCTION_FORK:-"illinois-ceesd"} - -echo "EMIRGE_INSTALL_PATH=${EMIRGE_INSTALL_PATH}" -echo "PRODUCTION_FORK=$PRODUCTION_FORK" -echo "PRODUCTION_BRANCH=$PRODUCTION_BRANCH" - -# Install the version of mirgecom we wish to test from source -./install.sh --skip-clone --install-prefix=${EMIRGE_INSTALL_PATH}/ --conda-env=${EMIRGE_INSTALL_PATH}/mirgecom/conda-env.yml --pip-pkgs=${EMIRGE_INSTALL_PATH}/mirgecom/requirements.txt - -# Activate the environment (needed?) -source ${EMIRGE_INSTALL_PATH}/config/activate_env.sh -cd ${EMIRGE_INSTALL_PATH}/mirgecom - -# This junk is needed to be able to execute git commands properly -git config user.email "ci-runner@ci.machine.com" -git config user.name "CI Runner" - -# Making a dedicated production remote adds production forks -git remote add production https://github.com/${PRODUCTION_FORK}/mirgecom -git fetch production - -# Merge the production branch for testing the production drivers -git merge production/${PRODUCTION_BRANCH} --no-edit -# Pick up any requirements.txt -pip install -r requirements.txt -cd - diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 5517dc273..75c83a192 100755 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -1,20 +1,12 @@ #!/bin/bash set -x # -# This script is designed to help customize the production environemtn +# This script is designed to help customize the production environment # under which CEESD production capability is tested under a proposed change -# to illinois-ceesd/mirgecom@main. If necessary, the script sets the -# following vars: +# to illinois-ceesd/mirgecom@main. # -# The proposed changes to test may be in a fork, or a local branch. For -# forks, the environment config files should set: -# -# export DEVELOPMENT_FORK="" # the fork/home of the development -# export DEVELOPMENT_BRANCH="" -# -# The production capability to test against may be specified outright, or -# patched by the incoming development. The following vars control the -# production environment: +# The production capability may be in a CEESD-local mirgecom branch or in a +# fork, and is specified through the following settings: # export PRODUCTION_BRANCH="ci-test-production" # The base production branch to be installed by emirge # export PRODUCTION_FORK="" # The fork/home of production changes (if any) From b053d56fb4d5ec3768595f92082c48ce369a8979 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 23:08:58 -0500 Subject: [PATCH 363/385] Uncustomize production env. --- .ci-support/production-testing-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 75c83a192..130a3c53e 100755 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -8,7 +8,7 @@ set -x # The production capability may be in a CEESD-local mirgecom branch or in a # fork, and is specified through the following settings: # -export PRODUCTION_BRANCH="ci-test-production" # The base production branch to be installed by emirge +# export PRODUCTION_BRANCH="" # The base production branch to be installed by emirge # export PRODUCTION_FORK="" # The fork/home of production changes (if any) # # Multiple production drivers are supported. The user should provide a ':'-delimited From 8034ba5069448ee3cc597a3651435ce00fc6688b Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Fri, 29 Oct 2021 23:10:21 -0500 Subject: [PATCH 364/385] Uncustomize production env. --- .ci-support/production-testing-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh index 75c83a192..130a3c53e 100755 --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -8,7 +8,7 @@ set -x # The production capability may be in a CEESD-local mirgecom branch or in a # fork, and is specified through the following settings: # -export PRODUCTION_BRANCH="ci-test-production" # The base production branch to be installed by emirge +# export PRODUCTION_BRANCH="" # The base production branch to be installed by emirge # export PRODUCTION_FORK="" # The fork/home of production changes (if any) # # Multiple production drivers are supported. The user should provide a ':'-delimited From 0f3a18891a70db7699d1d1e8bb5141475b888ff3 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 29 Oct 2021 23:55:28 -0500 Subject: [PATCH 365/385] Make CI production testing more robust and add documentation (#545) Co-authored-by: Matthias Diener --- .ci-support/install-mirge-from-source.sh | 13 +++ .../merge-install-production-branch.sh | 39 +++++++++ ...stall.sh => production-drivers-install.sh} | 16 +--- .ci-support/production-drivers-run.sh | 16 ++++ .ci-support/production-install.sh | 63 -------------- .ci-support/production-testing-env.sh | 16 +--- .github/workflows/ci.yaml | 23 ++--- doc/development/pullrequests.rst | 86 +++++++++++++++++++ 8 files changed, 173 insertions(+), 99 deletions(-) create mode 100755 .ci-support/install-mirge-from-source.sh create mode 100755 .ci-support/merge-install-production-branch.sh rename .ci-support/{production-driver-install.sh => production-drivers-install.sh} (81%) mode change 100644 => 100755 create mode 100755 .ci-support/production-drivers-run.sh delete mode 100644 .ci-support/production-install.sh mode change 100644 => 100755 .ci-support/production-testing-env.sh diff --git a/.ci-support/install-mirge-from-source.sh b/.ci-support/install-mirge-from-source.sh new file mode 100755 index 000000000..bfa55dd08 --- /dev/null +++ b/.ci-support/install-mirge-from-source.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -x +# +# This script is intended to install mirgecom from an uninstalled +# mirgecom source gotten from a fresh clone. +# +EMIRGE_INSTALL_PATH=${1:-"."} + +echo "EMIRGE_INSTALL_PATH=${EMIRGE_INSTALL_PATH}" + +# Install the version of mirgecom we wish to test from source +./install.sh --skip-clone --install-prefix=${EMIRGE_INSTALL_PATH}/ --conda-env=${EMIRGE_INSTALL_PATH}/mirgecom/conda-env.yml --pip-pkgs=${EMIRGE_INSTALL_PATH}/mirgecom/requirements.txt + diff --git a/.ci-support/merge-install-production-branch.sh b/.ci-support/merge-install-production-branch.sh new file mode 100755 index 000000000..3733c1a7d --- /dev/null +++ b/.ci-support/merge-install-production-branch.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -x +# +# This script is designed to patch the CEESD production capability into +# a proposed change to illinois-ceesd/mirgecom@main. The script reads the +# environment config file `.ci-support/production-testing-env.sh`, that +# should set up the expected control variables. +# +# The production capability to test against may be specified outright, or +# patched by the incoming development. The following vars control the +# production environment: +# +# PRODUCTION_BRANCH = The production branch (default=y1-production) +# PRODUCTION_FORK = The production fork (default=illinois-ceesd) +# +MIRGE_HOME=${1:-"."} +PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"y1-production"} +PRODUCTION_FORK=${PRODUCTION_FORK:-"illinois-ceesd"} + +echo "MIRGE_HOME=${MIRGE_HOME}" +echo "PRODUCTION_FORK=$PRODUCTION_FORK" +echo "PRODUCTION_BRANCH=$PRODUCTION_BRANCH" + +cd ${MIRGE_HOME} +git status + +# This junk is needed to be able to execute git commands properly +git config user.email "ci-runner@ci.machine.com" +git config user.name "CI Runner" + +# Making a dedicated production remote adds production forks +git remote add production https://github.com/${PRODUCTION_FORK}/mirgecom +git fetch production + +# Merge the production branch for testing the production drivers +git merge production/${PRODUCTION_BRANCH} --no-edit +# Pick up any requirements.txt +pip install -r requirements.txt +cd - diff --git a/.ci-support/production-driver-install.sh b/.ci-support/production-drivers-install.sh old mode 100644 new mode 100755 similarity index 81% rename from .ci-support/production-driver-install.sh rename to .ci-support/production-drivers-install.sh index e01c71aca..f04273d88 --- a/.ci-support/production-driver-install.sh +++ b/.ci-support/production-drivers-install.sh @@ -12,19 +12,9 @@ # The default values result in an install of the Y1 nozzle driver and # Wyatt Hagen's isolator driver that work with current MIRGE-Com # production branch: mirgecom@y1-production. -# -set -x - -DEVELOPMENT_BRANCH="$GITHUB_HEAD_REF" # this will be empty for main -PRODUCTION_DRIVERS="" -if [ -n "$DEVELOPMENT_BRANCH" ]; then - PRODUCTION_ENV_FILE="$1" - if [ -e "$PRODUCTION_ENV_FILE" ]; then - . $PRODUCTION_ENV_FILE - fi -fi -# Set to default if testing main, or user left it empty PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@y2-production:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar"} +# Loop over the production drivers, clone them, and prepare for execution +set -x OIFS="$IFS" IFS=':'; for production_driver_string in $PRODUCTION_DRIVERS; do @@ -34,7 +24,7 @@ do PRODUCTION_DRIVER_DIR="production_driver_$PRODUCTION_DRIVER_NAME" git clone -b "$PRODUCTION_DRIVER_BRANCH" https\://github.com/"$PRODUCTION_DRIVER_REPO" "$PRODUCTION_DRIVER_DIR" cd "$PRODUCTION_DRIVER_DIR"/smoke_test - ln -s *.py driver.py + ln -s *.py driver.py # name the driver generically cd ../.. done IFS="$OIFS" diff --git a/.ci-support/production-drivers-run.sh b/.ci-support/production-drivers-run.sh new file mode 100755 index 000000000..33d0fd89f --- /dev/null +++ b/.ci-support/production-drivers-run.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -x +# This script is designed to run the CEESD "production" drivers after +# they have been prepared by an external helper script called +# production-drivers-install.sh. The drivers are each expected to be +# in a directory called "production_driver_*" and are expected to have +# a test driver in "production_driver_*/smoke_test/driver.py". +DRIVERS_HOME=${1:-"."} +cd ${DRIVERS_HOME} +for production_driver in $(ls | grep "production_driver_"); +do + cd "$production_driver"/smoke_test + python -m mpi4py ./driver.py -i run_params.yaml + cd ../../ +done +cd - diff --git a/.ci-support/production-install.sh b/.ci-support/production-install.sh deleted file mode 100644 index 00b7f228c..000000000 --- a/.ci-support/production-install.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# -# This script is designed to patch the CEESD production capability against -# a proposed change to illinois-ceesd/mirgecom@main. The script reads the -# environment config file `.ci-support/production-testing-env.sh`, that -# should set up the expected control variables, which are as follows: -# -# The proposed changes to test may be in a fork, or a local branch. For -# forks, the environment config files should set: -# -# DEVELOPMENT_FORK = The development fork (default=illinois-ceesd) -# -# The production capability to test against may be specified outright, or -# patched by the incoming development. The following vars control the -# production environment: -# -# PRODUCTION_BRANCH = The production branch (default=y1-production) -# PRODUCTION_FORK = The production fork (default=illinois-ceesd) -# -# If the environment file does not exist, the current development is -# tested against `mirgecom@y1-production`. -# -set -x - -if [ -n "$DEVELOPMENT_BRANCH" ]; then - PRODUCTION_ENV_FILE="$1" - if [ -e "$PRODUCTION_ENV_FILE" ]; then - echo "Reading production configuration for ${DEVELOPMENT_BRANCH}." - . $PRODUCTION_ENV_FILE - else - echo "Using default production configuration for ${DEVELOPMENT_BRANCH}." - echo "To customize, set up .ci-support/production-testing-env.sh." - fi -fi -DEVELOPMENT_BRANCH=${DEVELOPMENT_BRANCH:-"main"} -DEVELOPMENT_FORK=${DEVELOPMENT_FORK:-"illinois-ceesd"} -PRODUCTION_BRANCH=${PRODUCTION_BRANCH:-"y1-production"} -PRODUCTION_FORK=${PRODUCTION_FORK:-"illinois-ceesd"} - -echo "Production environment settings:" -if [ -n "${PRODUCTION_ENV_FILE}" ]; then - echo "PRODUCTION_ENV_FILE=$PRODUCTION_ENV_FILE" - cat ${PRODUCTION_ENV_FILE} -fi -echo "DEVELOPMENT_FORK=$DEVELOPMENT_FORK" -echo "DEVELOPMENT_BRANCH=$DEVELOPMENT_BRANCH" -echo "PRODUCTION_FORK=$PRODUCTION_FORK" -echo "PRODUCTION_BRANCH=$PRODUCTION_BRANCH" - -# Install the production branch with emirge -./install.sh --fork=${DEVELOPMENT_FORK} --branch=${DEVELOPMENT_BRANCH} -source config/activate_env.sh -cd mirgecom - -# This junk is needed to be able to execute git commands properly -git config user.email "ci-runner@ci.machine.com" -git config user.name "CI Runner" - -# Merge in the production environment -git remote add production https://github.com/${PRODUCTION_FORK}/mirgecom -git fetch production -git merge production/${PRODUCTION_BRANCH} --no-edit -pip install -r requirements.txt diff --git a/.ci-support/production-testing-env.sh b/.ci-support/production-testing-env.sh old mode 100644 new mode 100755 index 6cf17aa41..130a3c53e --- a/.ci-support/production-testing-env.sh +++ b/.ci-support/production-testing-env.sh @@ -1,20 +1,12 @@ #!/bin/bash set -x # -# This script is designed to help customize the production environemtn +# This script is designed to help customize the production environment # under which CEESD production capability is tested under a proposed change -# to illinois-ceesd/mirgecom@main. If necessary, the script sets the -# following vars: +# to illinois-ceesd/mirgecom@main. # -# The proposed changes to test may be in a fork, or a local branch. For -# forks, the environment config files should set: -# -# export DEVELOPMENT_FORK="" # the fork/home of the development -# export DEVELOPMENT_BRANCH="" -# -# The production capability to test against may be specified outright, or -# patched by the incoming development. The following vars control the -# production environment: +# The production capability may be in a CEESD-local mirgecom branch or in a +# fork, and is specified through the following settings: # # export PRODUCTION_BRANCH="" # The base production branch to be installed by emirge # export PRODUCTION_FORK="" # The fork/home of production changes (if any) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index db21db82d..00b27a18d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -164,23 +164,24 @@ jobs: steps: - uses: actions/checkout@v2 + with: + fetch-depth: '0' - name: Prepare production environment run: | [[ $(uname) == Linux ]] && sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev [[ $(uname) == Darwin ]] && brew update && brew install mpich + MIRGEDIR=$(pwd) + cat .ci-support/production-testing-env.sh + . .ci-support/production-testing-env.sh cd .. git clone https://github.com/illinois-ceesd/emirge emirge.y1 cd emirge.y1 - . ../mirgecom/.ci-support/production-testing-env.sh - . ../mirgecom/.ci-support/production-install.sh ../mirgecom/.ci-support/production-testing-env.sh + . ../mirgecom/.ci-support/install-mirge-from-source.sh ${MIRGEDIR}/.. + source ${MIRGEDIR}/../config/activate_env.sh + . ../mirgecom/.ci-support/merge-install-production-branch.sh ${MIRGEDIR} - name: Run production test run: | - cd .. - source emirge.y1/config/activate_env.sh - . mirgecom/.ci-support/production-driver-install.sh mirgecom/.ci-support/production-testing-env.sh - for production_driver in $(ls | grep "production_driver_"); - do - cd "$production_driver"/smoke_test - python -m mpi4py ./driver.py -i run_params.yaml - cd ../../ - done + . .ci-support/production-testing-env.sh + source ../config/activate_env.sh + . .ci-support/production-drivers-install.sh . + . .ci-support/production-drivers-run.sh . diff --git a/doc/development/pullrequests.rst b/doc/development/pullrequests.rst index a34ef9c03..c670d139f 100644 --- a/doc/development/pullrequests.rst +++ b/doc/development/pullrequests.rst @@ -206,6 +206,92 @@ commit, you can use a git hook such as the following one (save this script as While highly recommended, hooks can sometimes be annoying. After setting up your hooks, you can use ``git --no-verify`` or equivalently ``git -n`` to run ``git`` commands without triggering the hooks. +Production Testing in CI +------------------------ + +The CI testing in mirgecom includes a set of "production" tests which help +detect when a proposed change in a PR breaks the CEESD prediction capability +toolchain. Most developments and PRs do not require special considerations +for the production tests, but any production test failures will require +a bit of care to resolve. + +When PRs run afoul of the CI production tests, it indicates that if the PR +change set merges to main, then the "production" capability of mirgecom will +not function until the production capability and the change set are brought +into accordance. + +To resolve CI production test failures for a development in PR, it is often useful +to run the production tests manually. The production tests may be prepared and +executed from anywhere by hand-executing the production test scripts found in +``.ci-support/``. The following is an example workflow adjacent to what CI itself +does for executing the production tests. + +1. Check out the PR development (and optionally make a production branch) + + The PR development is assumed to be in a mirgecom branch called ``branch-name`` + and possibly in a fork called ``fork-name``. + + .. code:: bash + + $ # For ceesd-local branches: + $ git clone -b branch-name git@github.com:/illinois-ceesd/mirgecom + $ # Or for developer fork: + $ git clone -b branch-name git@github.com:/fork-name/mirgecom + $ cd mirgecom # or loopy, meshmode, ... + $ git switch -c branch-name-production # Optional production branch + +2. Set up the production environment and capability + + .. code:: bash + + $ # Load the customizable production environment + $ . .ci-support/production-testing-env.sh + $ # Merge the production branch + $ . .ci-support/merge-install-production-branch.sh . + + If Step 2 fails, i.e. if there are merge conflicts, then those must + be resolved. Push the merged result to CEESD or a fork, and indicate + that version in the PRODUCTION_FORK, and PRODUCTION_BRANCH env vars in + ``.ci-support/production-testing-env.sh``. + +3. Grab and run the production tests + + .. code:: bash + + $ # Load env if needed + $ . .ci-support/production-testing-env.sh + $ # Get the production tests + $ . .ci-support/production-drivers-install.sh . + $ # Run the production tests + $ . .ci-support/production-drivers-run.sh . + + Step 3 will clone the production driver repos to the current directory, + with each driver in its own directory. If any of the drivers fail to + work with the current development, then they may be modified into working + condition and then pushed to a repo/branch. Indicate the location of the working + drivers in the PRODUCTION_DRIVERS env var customization in + ``.ci-support/production-testing-env.sh``. + +4. Update the PR to reflect the change in production environment (if any) + + Push the customized production ``.ci-support/production-testing-env.sh`` + settings to the PR development branch. Upon push, mirgecom CI will + try the production tests again, now with the customized environment. + + +If the PR development requires production environment customization in order to +pass production tests, then care and coordination will be required to get these +changes merged into the main mirgecom development. PR reviewers will help +with this process. If the PR is accepted for merging to main, then mirgecom +developers will update the production capabilities to be compatible with +the changes before merging. + + .. important:: + + Any production environment customizations must be backed out before + merging the PR development to main. Never merge a PR development with + production environment customizations in-place. + Merging a pull request ---------------------- From 3642ca5f710c39e06ca85233a381604c921e4735 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 1 Nov 2021 11:53:08 -0500 Subject: [PATCH 366/385] Update integrators to handle reference state --- mirgecom/integrators/__init__.py | 4 ++-- mirgecom/integrators/explicit_rk.py | 17 ++++++++++++----- mirgecom/integrators/lsrk.py | 27 +++++++++++++++++---------- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/mirgecom/integrators/__init__.py b/mirgecom/integrators/__init__.py index 6e00304c4..2a551666b 100644 --- a/mirgecom/integrators/__init__.py +++ b/mirgecom/integrators/__init__.py @@ -33,9 +33,9 @@ """ -def lsrk4_step(state, t, dt, rhs): +def lsrk4_step(state, t, dt, rhs, reference_state=None): """Call lsrk54_step with backwards-compatible interface.""" from warnings import warn warn("Do not call lsrk4; it is now callled lsrk54_step. This function will " "disappear August 1, 2021", DeprecationWarning, stacklevel=2) - return lsrk54_step(state, t, dt, rhs) + return lsrk54_step(state, t, dt, rhs, reference_state) diff --git a/mirgecom/integrators/explicit_rk.py b/mirgecom/integrators/explicit_rk.py index 5fe752cb7..51a5e3fa8 100644 --- a/mirgecom/integrators/explicit_rk.py +++ b/mirgecom/integrators/explicit_rk.py @@ -28,11 +28,18 @@ """ -def rk4_step(state, t, dt, rhs): +def rk4_step(state, t, dt, rhs, reference_state=None): """Take one step using the fourth-order Classical Runge-Kutta method.""" - k1 = rhs(t, state) - k2 = rhs(t+dt/2, state + dt/2*k1) - k3 = rhs(t+dt/2, state + dt/2*k2) - k4 = rhs(t+dt, state + dt*k3) + if reference_state is not None: + reference_state = state + k1 = rhs(t, state, reference_state) + k2 = rhs(t+dt/2, state + dt/2*k1, reference_state) + k3 = rhs(t+dt/2, state + dt/2*k2, reference_state) + k4 = rhs(t+dt, state + dt*k3, reference_state) + else: + k1 = rhs(t, state) + k2 = rhs(t+dt/2, state + dt/2*k1) + k3 = rhs(t+dt/2, state + dt/2*k2) + k4 = rhs(t+dt, state + dt*k3) return state + dt/6*(k1 + 2*k2 + 2*k3 + k4) diff --git a/mirgecom/integrators/lsrk.py b/mirgecom/integrators/lsrk.py index a1409b4a3..d50a9105a 100644 --- a/mirgecom/integrators/lsrk.py +++ b/mirgecom/integrators/lsrk.py @@ -46,12 +46,18 @@ class LSRKCoefficients: C: np.ndarray -def lsrk_step(coefs, state, t, dt, rhs): +def lsrk_step(coefs, state, t, dt, rhs, reference_state=None): """Take one step using a low-storage Runge-Kutta method.""" k = 0.0 * state - for i in range(len(coefs.A)): - k = coefs.A[i]*k + dt*rhs(t + coefs.C[i]*dt, state) - state += coefs.B[i]*k + if reference_state is not None: + reference_state = state + for i in range(len(coefs.A)): + k = coefs.A[i]*k + dt*rhs(t + coefs.C[i]*dt, state, reference_state) + state += coefs.B[i]*k + else: + for i in range(len(coefs.A)): + k = coefs.A[i]*k + dt*rhs(t + coefs.C[i]*dt, state) + state += coefs.B[i]*k return state @@ -62,9 +68,9 @@ def lsrk_step(coefs, state, t, dt, rhs): C=np.array([0.])) -def euler_step(state, t, dt, rhs): +def euler_step(state, t, dt, rhs, reference_state=None): """Take one step using the explicit, 1st-order accurate, Euler method.""" - return lsrk_step(EulerCoefs, state, t, dt, rhs) + return lsrk_step(EulerCoefs, state, t, dt, rhs, reference_state) LSRK54CarpenterKennedyCoefs = LSRKCoefficients( @@ -88,12 +94,12 @@ def euler_step(state, t, dt, rhs): 2802321613138/2924317926251])) -def lsrk54_step(state, t, dt, rhs): +def lsrk54_step(state, t, dt, rhs, reference_state=None): """Take one step using an explicit 5-stage, 4th-order, LSRK method. Coefficients are summarized in [Hesthaven_2008]_, Section 3.4. """ - return lsrk_step(LSRK54CarpenterKennedyCoefs, state, t, dt, rhs) + return lsrk_step(LSRK54CarpenterKennedyCoefs, state, t, dt, rhs, reference_state) LSRK144NiegemannDiehlBuschCoefs = LSRKCoefficients( @@ -144,11 +150,12 @@ def lsrk54_step(state, t, dt, rhs): 0.8734213127600976])) -def lsrk144_step(state, t, dt, rhs): +def lsrk144_step(state, t, dt, rhs, reference_state=None): """Take one step using an explicit 14-stage, 4th-order, LSRK method. This method is derived by Niegemann, Diehl, and Busch (2012), with an optimal stability region for advection-dominated flows. The LSRK coefficients are summarized in [Niegemann_2012]_, Table 3. """ - return lsrk_step(LSRK144NiegemannDiehlBuschCoefs, state, t, dt, rhs) + return lsrk_step(LSRK144NiegemannDiehlBuschCoefs, state, t, dt, rhs, + reference_state) From c76a91b71e583da71ad28ceb537f8626a83dbb9b Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 1 Nov 2021 11:54:29 -0500 Subject: [PATCH 367/385] update to use reference state for seeding temperature --- examples/autoignition-mpi.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index dd5121137..874f4cdbb 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -259,6 +259,13 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, from pytools.obj_array import make_obj_array + def get_temperature_update(state, temperature): + y = state.species_mass_fractions + e = eos.internal_energy(state) / state.mass + return make_obj_array( + [pyro_mechanism.get_temperature_update_energy(e, temperature, y)] + ) + def get_temperature_mass_energy(state, temperature): y = state.species_mass_fractions e = eos.internal_energy(state) / state.mass @@ -266,6 +273,7 @@ def get_temperature_mass_energy(state, temperature): [pyro_mechanism.get_temperature(e, temperature, y)] ) + compute_temperature_update = actx.compile(get_temperature_update) compute_dependent_vars = actx.compile(eos.dependent_vars) compute_temperature = actx.compile(get_temperature_mass_energy) @@ -292,6 +300,7 @@ def get_temperature_mass_energy(state, temperature): logmgr_set_time(logmgr, current_step, current_t) if order == rst_order: current_state = restart_data["state"] + rst_temperature = restart_data["temperature"] else: rst_state = restart_data["state"] old_discr = EagerDGDiscretization(actx, local_mesh, order=rst_order, @@ -300,9 +309,14 @@ def get_temperature_mass_energy(state, temperature): connection = make_same_mesh_connection(actx, discr.discr_from_dd("vol"), old_discr.discr_from_dd("vol")) current_state = connection(rst_state) + rst_temperature = connection(restart_data["temperature"]) + # This bit memoizes the restarted temperature onto the current_state + rst_temperature = compute_temperature(current_state, rst_temperature) else: # Set the current state from time 0 current_state = initializer(eos=eos, x_vec=nodes) + reference_state = current_state + # import ipdb # ipdb.set_trace() @@ -381,9 +395,11 @@ def my_write_restart(step, t, state): if rank == 0: logger.info("Skipping overwrite of restart file.") else: + rst_dv = compute_dependent_vars(state) rst_data = { "local_mesh": local_mesh, "state": state, + "temperature": rst_dv.temperature, "t": t, "step": step, "order": order, @@ -427,11 +443,10 @@ def my_health_check(cv, dv): # convergence in Pyrometheus `get_temperature`. # Note: The local max jig below works around a very long compile # in lazy mode. - check_temp, = compute_temperature(cv, temperature) - check_temp = thaw(freeze(check_temp, actx), actx) - temp_resid = actx.np.abs(check_temp - temperature) - temp_resid = op.nodal_max_loc(discr, "vol", temp_resid) - if temp_resid > 1e-12: + temp_update, = compute_temperature_update(cv, temperature) + temp_resid = thaw(freeze(temp_update, actx), actx) / temperature + temp_resid = (actx.to_numpy(op.nodal_max_loc(discr, "vol", temp_resid))) + if temp_resid > 1e-8: health_error = True logger.info(f"{rank=}: Temperature is not converged {temp_resid=}.") @@ -527,15 +542,16 @@ def my_pre_step(step, t, dt, state): def my_post_step(step, t, dt, state): # Logmgr needs to know about EOS, dt, dim? # imo this is a design/scope flaw - from mirgecom.simutil import limit_species_mass_fractions - state = limit_species_mass_fractions(state) if logmgr: set_dt(logmgr, dt) set_sim_state(logmgr, dim, state, eos) logmgr.tick_after() return state, dt - def my_rhs(t, state): + def my_rhs(t, state, reference_state): + current_dv = eos.dependent_vars(state, reference_state) + if debug: + print(f"{current_dv=}") return (euler_operator(discr, cv=state, time=t, boundaries=boundaries, eos=eos) + eos.get_species_source_terms(state)) @@ -547,7 +563,8 @@ def my_rhs(t, state): advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, - state=current_state, t=current_t, t_final=t_final) + state=current_state, t=current_t, t_final=t_final, + reference_state=reference_state) # Dump the final data if rank == 0: From a7b35750807b26f0893c7d503b8cc1ed8aed4e93 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 1 Nov 2021 11:55:59 -0500 Subject: [PATCH 368/385] Update to use reference state --- mirgecom/eos.py | 47 ++++++++++++++++++++++++++++++++++++++------ mirgecom/steppers.py | 12 +++++++---- 2 files changed, 49 insertions(+), 10 deletions(-) diff --git a/mirgecom/eos.py b/mirgecom/eos.py index bd18333cf..5773f0690 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -90,7 +90,8 @@ def pressure(self, cv: ConservedVars): """Get the gas pressure.""" @abstractmethod - def temperature(self, cv: ConservedVars): + def temperature(self, cv: ConservedVars, + reference_state: ConservedVars = None): """Get the gas temperature.""" @abstractmethod @@ -133,11 +134,12 @@ def transport_model(self): def get_internal_energy(self, temperature, *, mass, species_mass_fractions): """Get the fluid internal energy from temperature and mass.""" - def dependent_vars(self, cv: ConservedVars) -> EOSDependentVars: + def dependent_vars(self, cv: ConservedVars, + reference_state: ConservedVars = None) -> EOSDependentVars: """Get an agglomerated array of the dependent variables.""" return EOSDependentVars( pressure=self.pressure(cv), - temperature=self.temperature(cv), + temperature=self.temperature(cv, reference_state), ) @@ -152,8 +154,14 @@ class MixtureEOS(GasEOS): .. automethod:: get_production_rates .. automethod:: species_enthalpies .. automethod:: get_species_source_terms + .. automethod:: get_temperature_guess """ + @abstractmethod + def get_temperature_guess(self, cv: ConservedVars, + reference_state: ConservedVars = None): + r"""Get a constant and uniform guess for the gas temperature.""" + @abstractmethod def get_density(self, pressure, temperature, species_mass_fractions): """Get the density from pressure, temperature, and species fractions (Y).""" @@ -332,7 +340,7 @@ def sound_speed(self, cv: ConservedVars): actx = cv.array_context return actx.np.sqrt(self._gamma / cv.mass * self.pressure(cv)) - def temperature(self, cv: ConservedVars): + def temperature(self, cv: ConservedVars, reference_state: ConservedVars = None): r"""Get the thermodynamic temperature of the gas. The thermodynamic temperature (T) is calculated from @@ -349,6 +357,10 @@ def temperature(self, cv: ConservedVars): :class:`~mirgecom.fluid.ConservedVars` containing at least the mass ($\rho$), energy ($\rho{E}$), momentum ($\rho\vec{V}$). + reference_state: :class:`~mirgecom.fluid.ConservedVars` + :class:`~mirgecom.fluid.ConservedVars` from which to seed temperature + calculation. + Returns ------- :class:`~meshmode.dof_array.DOFArray` @@ -479,6 +491,28 @@ def __init__(self, pyrometheus_mech, temperature_guess=300.0, self._tguess = temperature_guess self._transport_model = transport_model + def get_temperature_guess(self, cv, reference_state=None): + """Get a constant and uniform guess for the gas temperature. + + Parameters + ---------- + cv: :class:`~mirgecom.fluid.ConservedVars` + :class:`~mirgecom.fluid.ConservedVars` used to conjure the required shape + for the returned temperature guess. + reference_state: :class:`~mirgecom.fluid.ConservedVars` + :class:`~mirgecom.fluid.ConservedVars` optionally seed the temperature + calculation with the temperature from this state + + Returns + ------- + :class:`~meshmode.dof_array.DOFArray` + The temperature with which to seed the Newton solver in + :module:thermochemistry. + """ + if reference_state is not None: + return self.temperature(reference_state) # return already-memoized temp + return (self._tguess * (0*cv.mass + 1.0)) + def transport_model(self): """Get the transport model object for this EOS.""" return self._transport_model @@ -723,7 +757,7 @@ def get_sos(): return actx.np.sqrt((self.gamma(cv) * self.pressure(cv)) / cv.mass) return get_sos() - def temperature(self, cv: ConservedVars): + def temperature(self, cv: ConservedVars, reference_state=None): r"""Get the thermodynamic temperature of the gas. The thermodynamic temperature ($T$) is calculated from @@ -751,10 +785,11 @@ def temperature(self, cv: ConservedVars): @memoize_in(cv, (PyrometheusMixture.temperature, type(self._pyrometheus_mech))) def get_temp(): - tguess = self._tguess + 0*cv.mass + tguess = self.get_temperature_guess(cv, reference_state=reference_state) y = cv.species_mass_fractions e = self.internal_energy(cv) / cv.mass return self._pyrometheus_mech.get_temperature(e, tguess, y) + return get_temp() def total_energy(self, cv, pressure): diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index e14f513bb..d668c7f1f 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -63,7 +63,7 @@ def _advance_state_stepper_func(rhs, timestepper, pre_step_callback=None, post_step_callback=None, logmgr=None, eos=None, dim=None, - actx=None): + actx=None, reference_state=None): """Advance state from some time (t) to some time (t_final). Parameters @@ -123,7 +123,11 @@ def _advance_state_stepper_func(rhs, timestepper, if pre_step_callback is not None: state, dt = pre_step_callback(state=state, step=istep, t=t, dt=dt) - state = timestepper(state=state, t=t, dt=dt, rhs=compiled_rhs) + if reference_state is not None: + reference_state = state + + state = timestepper(state=state, t=t, dt=dt, rhs=compiled_rhs, + reference_state=reference_state) t += dt istep += 1 @@ -272,7 +276,7 @@ def advance_state(rhs, timestepper, state, t_final, pre_step_callback=None, post_step_callback=None, logmgr=None, eos=None, dim=None, - actx=None): + actx=None, reference_state=None): """Determine what stepper we're using and advance the state from (t) to (t_final). Parameters @@ -357,7 +361,7 @@ def advance_state(rhs, timestepper, state, t_final, pre_step_callback=pre_step_callback, post_step_callback=post_step_callback, istep=istep, logmgr=logmgr, eos=eos, dim=dim, - actx=actx + actx=actx, reference_state=reference_state ) return current_step, current_t, current_state From 0d0c040e50d1fc4cde62faaf946d2345a2726978 Mon Sep 17 00:00:00 2001 From: Michael Campbell Date: Mon, 1 Nov 2021 11:57:12 -0500 Subject: [PATCH 369/385] Publicize temperature update routine for use in health checking. --- mirgecom/thermochemistry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirgecom/thermochemistry.py b/mirgecom/thermochemistry.py index 81bbb9438..081cc28a7 100644 --- a/mirgecom/thermochemistry.py +++ b/mirgecom/thermochemistry.py @@ -67,7 +67,7 @@ def get_concentrations(self, rho, mass_fractions): return concs # This is the temperature update for *get_temperature* - def _get_temperature_update_energy(self, e_in, t_in, y): + def get_temperature_update_energy(self, e_in, t_in, y): pv_func = self.get_mixture_specific_heat_cv_mass he_func = self.get_mixture_internal_energy_mass return (e_in - he_func(t_in, y)) / pv_func(t_in, y) @@ -99,7 +99,7 @@ def get_temperature(self, energy, temperature_guess, species_mass_fractions): num_iter = temperature_niter t_i = temperature_guess for _ in range(num_iter): - t_i = t_i + self._get_temperature_update_energy( + t_i = t_i + self.get_temperature_update_energy( energy, t_i, species_mass_fractions ) return t_i From 0c56fb1d966719582e1cbbb94bff9ac7cd8756da Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Tue, 2 Nov 2021 16:18:34 -0500 Subject: [PATCH 370/385] rename _evaluate_state -> _force_evaluation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andreas Klöckner --- mirgecom/steppers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 3ff5460eb..a8f129487 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -64,7 +64,7 @@ def get_rhs(): return get_rhs() -def _evaluate_state(actx, state): +def _force_evaluation(actx, state): if actx is None: return state return thaw(freeze(state, actx), actx) @@ -126,7 +126,7 @@ def _advance_state_stepper_func(rhs, timestepper, compiled_rhs = _compile_rhs(actx, rhs) while t < t_final: - state = _evaluate_state(actx, state) + state = _force_evaluation(actx, state) if logmgr: logmgr.tick_before() @@ -207,7 +207,7 @@ def _advance_state_leap(rhs, timestepper, state, compiled_rhs, t, dt, state) while t < t_final: - state = _evaluate_state(actx, state) + state = _force_evaluation(actx, state) if pre_step_callback is not None: state, dt = pre_step_callback(state=state, From 503a94007f87883f42569b47565b3edaa367d80c Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Wed, 3 Nov 2021 14:57:29 -0500 Subject: [PATCH 371/385] Remove explicit `actx` argument from `advance_state` (#530) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove explicit actx argument from advance_state * rename _evaluate_state -> _force_evaluation Co-authored-by: Andreas Klöckner Co-authored-by: Andreas Klöckner --- mirgecom/steppers.py | 50 ++++++++++++++++++----------------- test/test_time_integrators.py | 6 ++--- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index e14f513bb..a8f129487 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -32,10 +32,14 @@ from logpyle import set_dt from mirgecom.logging_quantities import set_sim_state from pytools import memoize_in -from arraycontext import freeze, thaw +from arraycontext import ( + freeze, + thaw, + get_container_context_recursively +) -def compile_timestepper(actx, timestepper, state, rhs): +def _compile_timestepper(actx, timestepper, rhs): """Create lazy evaluation version of the timestepper.""" @memoize_in(actx, ("mirgecom_compiled_operator", timestepper, rhs)) @@ -47,8 +51,11 @@ def get_timestepper(): return get_timestepper() -def compile_rhs(actx, rhs, state): +def _compile_rhs(actx, rhs): """Create lazy evaluation version of the rhs.""" + if actx is None: + return rhs + @memoize_in(actx, ("mirgecom_compiled_rhs", rhs)) def get_rhs(): @@ -57,13 +64,18 @@ def get_rhs(): return get_rhs() +def _force_evaluation(actx, state): + if actx is None: + return state + return thaw(freeze(state, actx), actx) + + def _advance_state_stepper_func(rhs, timestepper, state, t_final, dt=0, t=0.0, istep=0, pre_step_callback=None, post_step_callback=None, - logmgr=None, eos=None, dim=None, - actx=None): + logmgr=None, eos=None, dim=None): """Advance state from some time (t) to some time (t_final). Parameters @@ -109,13 +121,12 @@ def _advance_state_stepper_func(rhs, timestepper, if t_final <= t: return istep, t, state - if actx is None: - actx = state.array_context + actx = get_container_context_recursively(state) - compiled_rhs = compile_rhs(actx, rhs, state) + compiled_rhs = _compile_rhs(actx, rhs) while t < t_final: - state = thaw(freeze(state, actx), actx) + state = _force_evaluation(actx, state) if logmgr: logmgr.tick_before() @@ -145,8 +156,7 @@ def _advance_state_leap(rhs, timestepper, state, t=0.0, istep=0, pre_step_callback=None, post_step_callback=None, - logmgr=None, eos=None, dim=None, - actx=None): + logmgr=None, eos=None, dim=None): """Advance state from some time *t* to some time *t_final* using :mod:`leap`. Parameters @@ -190,19 +200,14 @@ def _advance_state_leap(rhs, timestepper, state, if t_final <= t: return istep, t, state - if actx is None: - actx = state.array_context + actx = get_container_context_recursively(state) - compiled_rhs = compile_rhs(actx, rhs, state) + compiled_rhs = _compile_rhs(actx, rhs) stepper_cls = generate_singlerate_leap_advancer(timestepper, component_id, compiled_rhs, t, dt, state) - while t < t_final: - # This is only needed because Leap testing in test/test_time_integrators.py - # tests on single scalar values rather than an array-context-ready array - # container like a CV. - if isinstance(state, np.ndarray): - state = thaw(freeze(state, actx), actx) + while t < t_final: + state = _force_evaluation(actx, state) if pre_step_callback is not None: state, dt = pre_step_callback(state=state, @@ -271,8 +276,7 @@ def advance_state(rhs, timestepper, state, t_final, t=0.0, istep=0, dt=0, pre_step_callback=None, post_step_callback=None, - logmgr=None, eos=None, dim=None, - actx=None): + logmgr=None, eos=None, dim=None): """Determine what stepper we're using and advance the state from (t) to (t_final). Parameters @@ -347,7 +351,6 @@ def advance_state(rhs, timestepper, state, t_final, post_step_callback=post_step_callback, component_id=component_id, istep=istep, logmgr=logmgr, eos=eos, dim=dim, - actx=actx ) else: (current_step, current_t, current_state) = \ @@ -357,7 +360,6 @@ def advance_state(rhs, timestepper, state, t_final, pre_step_callback=pre_step_callback, post_step_callback=post_step_callback, istep=istep, logmgr=logmgr, eos=eos, dim=dim, - actx=actx ) return current_step, current_t, current_state diff --git a/test/test_time_integrators.py b/test/test_time_integrators.py index a78ac1187..069469778 100644 --- a/test/test_time_integrators.py +++ b/test/test_time_integrators.py @@ -105,10 +105,8 @@ def rhs(t, state): (SSPRK22MethodBuilder("y"), 2), (SSPRK33MethodBuilder("y"), 3), ]) - def test_leapgen_integration_order(actx_factory, method, method_order): + def test_leapgen_integration_order(method, method_order): """Test that time integrators have correct order.""" - actx = actx_factory() - def exact_soln(t): return np.exp(-t) @@ -130,7 +128,7 @@ def rhs(t, y): (step, t, state) = \ advance_state(rhs=rhs, timestepper=method, dt=dt, state=state, t=t, t_final=t_final, - component_id="y", actx=actx) + component_id="y") error = np.abs(state - exact_soln(t)) / exact_soln(t) integrator_eoc.add_data_point(dt, error) From 4e3aaf511b9c01e29c936f7853ca4479e47f9bcf Mon Sep 17 00:00:00 2001 From: CI Runner Date: Thu, 4 Nov 2021 09:06:16 -0500 Subject: [PATCH 372/385] Refactor to incrementally centralize EOS functions --- examples/autoignition-mpi.py | 5 ++--- mirgecom/eos.py | 2 +- mirgecom/euler.py | 8 ++++++-- mirgecom/integrators/explicit_rk.py | 1 - mirgecom/inviscid.py | 29 +++++++++++++++-------------- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 874f4cdbb..9b936f809 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -550,10 +550,9 @@ def my_post_step(step, t, dt, state): def my_rhs(t, state, reference_state): current_dv = eos.dependent_vars(state, reference_state) - if debug: - print(f"{current_dv=}") return (euler_operator(discr, cv=state, time=t, - boundaries=boundaries, eos=eos) + boundaries=boundaries, eos=eos, + dv=current_dv) + eos.get_species_source_terms(state)) current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, diff --git a/mirgecom/eos.py b/mirgecom/eos.py index 5773f0690..4743e5bbc 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -138,8 +138,8 @@ def dependent_vars(self, cv: ConservedVars, reference_state: ConservedVars = None) -> EOSDependentVars: """Get an agglomerated array of the dependent variables.""" return EOSDependentVars( - pressure=self.pressure(cv), temperature=self.temperature(cv, reference_state), + pressure=self.pressure(cv), ) diff --git a/mirgecom/euler.py b/mirgecom/euler.py index b39675042..aaf07e7d9 100644 --- a/mirgecom/euler.py +++ b/mirgecom/euler.py @@ -66,7 +66,8 @@ from mirgecom.operators import div_operator -def euler_operator(discr, eos, boundaries, cv, time=0.0): +def euler_operator(discr, eos, boundaries, cv, time=0.0, + dv=None): r"""Compute RHS of the Euler flow equations. Returns @@ -100,7 +101,10 @@ def euler_operator(discr, eos, boundaries, cv, time=0.0): Agglomerated object array of DOF arrays representing the RHS of the Euler flow equations. """ - inviscid_flux_vol = inviscid_flux(discr, eos, cv) + if dv is None: + dv = eos.dependent_vars(cv) + + inviscid_flux_vol = inviscid_flux(discr, dv.pressure, cv) inviscid_flux_bnd = ( inviscid_facial_flux(discr, eos=eos, cv_tpair=interior_trace_pair(discr, cv)) + sum(inviscid_facial_flux( diff --git a/mirgecom/integrators/explicit_rk.py b/mirgecom/integrators/explicit_rk.py index 51a5e3fa8..11870c689 100644 --- a/mirgecom/integrators/explicit_rk.py +++ b/mirgecom/integrators/explicit_rk.py @@ -31,7 +31,6 @@ def rk4_step(state, t, dt, rhs, reference_state=None): """Take one step using the fourth-order Classical Runge-Kutta method.""" if reference_state is not None: - reference_state = state k1 = rhs(t, state, reference_state) k2 = rhs(t+dt/2, state + dt/2*k1, reference_state) k3 = rhs(t+dt/2, state + dt/2*k2, reference_state) diff --git a/mirgecom/inviscid.py b/mirgecom/inviscid.py index c04eaf578..dedf3f6f8 100644 --- a/mirgecom/inviscid.py +++ b/mirgecom/inviscid.py @@ -45,7 +45,7 @@ from mirgecom.fluid import make_conserved -def inviscid_flux(discr, eos, cv): +def inviscid_flux(discr, pressure, cv): r"""Compute the inviscid flux vectors from fluid conserved vars *cv*. The inviscid fluxes are @@ -59,16 +59,16 @@ def inviscid_flux(discr, eos, cv): :class:`mirgecom.fluid.ConservedVars` for more information about how the fluxes are represented. """ - dim = cv.dim - p = eos.pressure(cv) - - mom = cv.momentum - - return make_conserved( - dim, mass=mom, energy=mom * (cv.energy + p) / cv.mass, - momentum=np.outer(mom, mom) / cv.mass + np.eye(dim)*p, - species_mass=( # reshaped: (nspecies, dim) - (mom / cv.mass) * cv.species_mass.reshape(-1, 1))) + mass_flux = cv.momentum + energy_flux = cv.velocity * (cv.energy + pressure) + mom_flux = ( + cv.mass * np.outer(cv.velocity, cv.velocity) + np.eye(cv.dim)*pressure + ) + species_mass_flux = ( # reshaped: (nspeceis, dim) + cv.velocity * cv.species_mass.reshape(-1, 1) + ) + return make_conserved(cv.dim, mass=mass_flux, energy=energy_flux, + momentum=mom_flux, species_mass=species_mass_flux) def inviscid_facial_flux(discr, eos, cv_tpair, local=False): @@ -104,10 +104,11 @@ def inviscid_facial_flux(discr, eos, cv_tpair, local=False): "all_faces"; remaining instead on the boundary restriction. """ actx = cv_tpair.int.array_context - + p_int = eos.pressure(cv_tpair.int) + p_ext = eos.pressure(cv_tpair.ext) flux_tpair = TracePair(cv_tpair.dd, - interior=inviscid_flux(discr, eos, cv_tpair.int), - exterior=inviscid_flux(discr, eos, cv_tpair.ext)) + interior=inviscid_flux(discr, p_int, cv_tpair.int), + exterior=inviscid_flux(discr, p_ext, cv_tpair.ext)) # This calculates the local maximum eigenvalue of the flux Jacobian # for a single component gas, i.e. the element-local max wavespeed |v| + c. From 6cacf6c39399b8617f60dd9e0ee55db95ed4e332 Mon Sep 17 00:00:00 2001 From: CI Runner Date: Thu, 4 Nov 2021 12:06:29 -0500 Subject: [PATCH 373/385] Switch eos-side handling reference_state -> temperature_seed --- examples/autoignition-mpi.py | 59 +++++++++++++++++++++++------------- mirgecom/eos.py | 43 ++++++++++++++------------ mirgecom/steppers.py | 6 ++-- 3 files changed, 65 insertions(+), 43 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index 9b936f809..ccb45f72b 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -216,7 +216,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, # Initial temperature, pressure, and mixutre mole fractions are needed to # set up the initial state in Cantera. - init_temperature = 1500.0 # Initial temperature hot enough to burn + temperature_seed = 1500.0 # Initial temperature hot enough to burn # Parameters for calculating the amounts of fuel, oxidizer, and inert species equiv_ratio = 1.0 ox_di_ratio = 0.21 @@ -235,9 +235,9 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, # one_atm = 101325.0 # Let the user know about how Cantera is being initilized - print(f"Input state (T,P,X) = ({init_temperature}, {one_atm}, {x}") + print(f"Input state (T,P,X) = ({temperature_seed}, {one_atm}, {x}") # Set Cantera internal gas temperature, pressure, and mole fractios - cantera_soln.TPX = init_temperature, one_atm, x + cantera_soln.TPX = temperature_seed, one_atm, x # Pull temperature, total density, mass fractions, and pressure from Cantera # We need total density, and mass fractions to initialize the fluid/gas state. can_t, can_rho, can_y = cantera_soln.TDY @@ -255,7 +255,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True, # states for this particular mechanism. from mirgecom.thermochemistry import make_pyrometheus_mechanism_class pyro_mechanism = make_pyrometheus_mechanism_class(cantera_soln)(actx.np) - eos = PyrometheusMixture(pyro_mechanism, temperature_guess=init_temperature) + eos = PyrometheusMixture(pyro_mechanism, temperature_guess=temperature_seed) from pytools.obj_array import make_obj_array @@ -266,16 +266,8 @@ def get_temperature_update(state, temperature): [pyro_mechanism.get_temperature_update_energy(e, temperature, y)] ) - def get_temperature_mass_energy(state, temperature): - y = state.species_mass_fractions - e = eos.internal_energy(state) / state.mass - return make_obj_array( - [pyro_mechanism.get_temperature(e, temperature, y)] - ) - compute_temperature_update = actx.compile(get_temperature_update) compute_dependent_vars = actx.compile(eos.dependent_vars) - compute_temperature = actx.compile(get_temperature_mass_energy) # }}} @@ -300,7 +292,7 @@ def get_temperature_mass_energy(state, temperature): logmgr_set_time(logmgr, current_step, current_t) if order == rst_order: current_state = restart_data["state"] - rst_temperature = restart_data["temperature"] + temperature_seed = restart_data["temperature"] else: rst_state = restart_data["state"] old_discr = EagerDGDiscretization(actx, local_mesh, order=rst_order, @@ -309,13 +301,32 @@ def get_temperature_mass_energy(state, temperature): connection = make_same_mesh_connection(actx, discr.discr_from_dd("vol"), old_discr.discr_from_dd("vol")) current_state = connection(rst_state) - rst_temperature = connection(restart_data["temperature"]) - # This bit memoizes the restarted temperature onto the current_state - rst_temperature = compute_temperature(current_state, rst_temperature) + temperature_seed = connection(restart_data["temperature"]) else: # Set the current state from time 0 current_state = initializer(eos=eos, x_vec=nodes) - reference_state = current_state + + # This bit memoizes the initial state's temperature onto the initial state + # (assuming `initializer` just above didn't call eos.dv funcs.) + # + # The temperature_seed going into this function is: + # - At time 0: the initial temperature input data (maybe from Cantera) + # - On restart: the restarted temperature read from restart file + # + # Note that this means we *seed* the temperature calculation with the actual + # temperature from the run. The resulting temperature may be different from the + # seed (error commensurate with convergence of running temperature), potentially + # meaning non-deterministic temperature restarts (i.e. one where the code gets a + # slightly different answer for temperature than it would have without the + # restart). In the absense of restart, the running temperature is that which was + # computed with a temperature_seed that equals the running temperature from the + # last step. + # Potentially, we could move the restart writing to trigger at post_step_callback + # and instead of writing the *current* running temperature to the restart file, + # we could write the *temperature_seed*. That could fix up the non-deterministic + # restart issue. + temperature_seed = eos.temperature(current_state, + temperature_seed=temperature_seed) # import ipdb # ipdb.set_trace() @@ -490,7 +501,7 @@ def my_get_timestep(t, dt, state): comm=comm, op=MPI.MAX) return ts_field, cfl, min(t_remaining, dt) - def my_pre_step(step, t, dt, state): + def my_pre_step(step, t, dt, state, reference_state): try: dv = None @@ -539,7 +550,10 @@ def my_pre_step(step, t, dt, state): return state, dt - def my_post_step(step, t, dt, state): + def my_post_step(step, t, dt, state, reference_state): + ref_dv = compute_dependent_vars(reference_state) + new_dv = compute_dependent_vars(state, # noqa + temperature_seed=ref_dv.temperature) # Logmgr needs to know about EOS, dt, dim? # imo this is a design/scope flaw if logmgr: @@ -549,7 +563,10 @@ def my_post_step(step, t, dt, state): return state, dt def my_rhs(t, state, reference_state): - current_dv = eos.dependent_vars(state, reference_state) + ref_dv = eos.dependent_vars(reference_state) + current_dv = eos.dependent_vars(state, + temperature_seed=ref_dv.temperature) + return (euler_operator(discr, cv=state, time=t, boundaries=boundaries, eos=eos, dv=current_dv) @@ -563,7 +580,7 @@ def my_rhs(t, state, reference_state): pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, state=current_state, t=current_t, t_final=t_final, - reference_state=reference_state) + reference_state=current_state) # Dump the final data if rank == 0: diff --git a/mirgecom/eos.py b/mirgecom/eos.py index 4743e5bbc..f189383b3 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -41,6 +41,7 @@ import numpy as np from pytools import memoize_in from meshmode.mesh import BTAG_ALL, BTAG_NONE # noqa +from meshmode.dof_array import DOFArray from mirgecom.fluid import ConservedVars, make_conserved from abc import ABCMeta, abstractmethod from arraycontext import dataclass_array_container @@ -135,10 +136,10 @@ def get_internal_energy(self, temperature, *, mass, species_mass_fractions): """Get the fluid internal energy from temperature and mass.""" def dependent_vars(self, cv: ConservedVars, - reference_state: ConservedVars = None) -> EOSDependentVars: + temperature_seed: DOFArray = None) -> EOSDependentVars: """Get an agglomerated array of the dependent variables.""" return EOSDependentVars( - temperature=self.temperature(cv, reference_state), + temperature=self.temperature(cv, temperature_seed), pressure=self.pressure(cv), ) @@ -154,12 +155,12 @@ class MixtureEOS(GasEOS): .. automethod:: get_production_rates .. automethod:: species_enthalpies .. automethod:: get_species_source_terms - .. automethod:: get_temperature_guess + .. automethod:: get_temperature_seed """ @abstractmethod - def get_temperature_guess(self, cv: ConservedVars, - reference_state: ConservedVars = None): + def get_temperature_seed(self, cv: ConservedVars, + temperature_seed: DOFArray = None): r"""Get a constant and uniform guess for the gas temperature.""" @abstractmethod @@ -340,7 +341,7 @@ def sound_speed(self, cv: ConservedVars): actx = cv.array_context return actx.np.sqrt(self._gamma / cv.mass * self.pressure(cv)) - def temperature(self, cv: ConservedVars, reference_state: ConservedVars = None): + def temperature(self, cv: ConservedVars, temperature_seed: DOFArray = None): r"""Get the thermodynamic temperature of the gas. The thermodynamic temperature (T) is calculated from @@ -357,9 +358,8 @@ def temperature(self, cv: ConservedVars, reference_state: ConservedVars = None): :class:`~mirgecom.fluid.ConservedVars` containing at least the mass ($\rho$), energy ($\rho{E}$), momentum ($\rho\vec{V}$). - reference_state: :class:`~mirgecom.fluid.ConservedVars` - :class:`~mirgecom.fluid.ConservedVars` from which to seed temperature - calculation. + temperature_seed: float or :class:`~meshmode.dof_array.DOFArray` + Ignored for this EOS. Returns ------- @@ -462,6 +462,7 @@ class PyrometheusMixture(MixtureEOS): .. automethod:: get_production_rates .. automethod:: species_enthalpies .. automethod:: get_species_source_terms + .. automethod:: get_temperature_seed """ def __init__(self, pyrometheus_mech, temperature_guess=300.0, @@ -491,17 +492,16 @@ def __init__(self, pyrometheus_mech, temperature_guess=300.0, self._tguess = temperature_guess self._transport_model = transport_model - def get_temperature_guess(self, cv, reference_state=None): - """Get a constant and uniform guess for the gas temperature. + def get_temperature_seed(self, cv, temperature_seed=None): + """Get a *cv*-shape-consistent array with which to seed temperature calcuation. Parameters ---------- cv: :class:`~mirgecom.fluid.ConservedVars` :class:`~mirgecom.fluid.ConservedVars` used to conjure the required shape for the returned temperature guess. - reference_state: :class:`~mirgecom.fluid.ConservedVars` - :class:`~mirgecom.fluid.ConservedVars` optionally seed the temperature - calculation with the temperature from this state + temperature_seed: float or :class:`~meshmode.dof_array.DOFArray` + Optional data from which to seed temperature calculation. Returns ------- @@ -509,9 +509,10 @@ def get_temperature_guess(self, cv, reference_state=None): The temperature with which to seed the Newton solver in :module:thermochemistry. """ - if reference_state is not None: - return self.temperature(reference_state) # return already-memoized temp - return (self._tguess * (0*cv.mass + 1.0)) + tseed = self._tguess + if temperature_seed is not None: + tseed = temperature_seed + return tseed if isinstance(tseed, DOFArray) else tseed * (0*cv.mass + 1.0) def transport_model(self): """Get the transport model object for this EOS.""" @@ -757,7 +758,7 @@ def get_sos(): return actx.np.sqrt((self.gamma(cv) * self.pressure(cv)) / cv.mass) return get_sos() - def temperature(self, cv: ConservedVars, reference_state=None): + def temperature(self, cv: ConservedVars, temperature_seed=None): r"""Get the thermodynamic temperature of the gas. The thermodynamic temperature ($T$) is calculated from @@ -774,6 +775,8 @@ def temperature(self, cv: ConservedVars, reference_state=None): :class:`~mirgecom.fluid.ConservedVars` containing at least the mass ($\rho$), energy ($\rho{E}$), momentum ($\rho\vec{V}$), and the vector of species masses, ($\rho{Y}_\alpha$). + temperature_seed: float or :class:`~meshmode.dof_array.DOFArray` + Optional data from which to seed temperature calculation. Returns ------- @@ -785,10 +788,10 @@ def temperature(self, cv: ConservedVars, reference_state=None): @memoize_in(cv, (PyrometheusMixture.temperature, type(self._pyrometheus_mech))) def get_temp(): - tguess = self.get_temperature_guess(cv, reference_state=reference_state) + tseed = self.get_temperature_seed(cv, temperature_seed) y = cv.species_mass_fractions e = self.internal_energy(cv) / cv.mass - return self._pyrometheus_mech.get_temperature(e, tguess, y) + return self._pyrometheus_mech.get_temperature(e, tseed, y) return get_temp() diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index bf1927d2d..2ea10c478 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -133,7 +133,8 @@ def _advance_state_stepper_func(rhs, timestepper, logmgr.tick_before() if pre_step_callback is not None: - state, dt = pre_step_callback(state=state, step=istep, t=t, dt=dt) + state, dt = pre_step_callback(state=state, step=istep, t=t, dt=dt, + reference_state=reference_state) if reference_state is not None: reference_state = state @@ -145,7 +146,8 @@ def _advance_state_stepper_func(rhs, timestepper, istep += 1 if post_step_callback is not None: - state, dt = post_step_callback(state=state, step=istep, t=t, dt=dt) + state, dt = post_step_callback(state=state, step=istep, t=t, dt=dt, + reference_state=reference_state) if logmgr: set_dt(logmgr, dt) From 947a5460dd853c066bb9afaf5a601480cebae0ad Mon Sep 17 00:00:00 2001 From: CI Runner Date: Thu, 4 Nov 2021 13:46:01 -0500 Subject: [PATCH 374/385] Banish reference state from this machinery --- mirgecom/integrators/__init__.py | 4 ++-- mirgecom/integrators/explicit_rk.py | 16 +++++----------- mirgecom/integrators/lsrk.py | 28 ++++++++++------------------ mirgecom/steppers.py | 21 ++++++--------------- 4 files changed, 23 insertions(+), 46 deletions(-) diff --git a/mirgecom/integrators/__init__.py b/mirgecom/integrators/__init__.py index 2a551666b..6e00304c4 100644 --- a/mirgecom/integrators/__init__.py +++ b/mirgecom/integrators/__init__.py @@ -33,9 +33,9 @@ """ -def lsrk4_step(state, t, dt, rhs, reference_state=None): +def lsrk4_step(state, t, dt, rhs): """Call lsrk54_step with backwards-compatible interface.""" from warnings import warn warn("Do not call lsrk4; it is now callled lsrk54_step. This function will " "disappear August 1, 2021", DeprecationWarning, stacklevel=2) - return lsrk54_step(state, t, dt, rhs, reference_state) + return lsrk54_step(state, t, dt, rhs) diff --git a/mirgecom/integrators/explicit_rk.py b/mirgecom/integrators/explicit_rk.py index 11870c689..5fe752cb7 100644 --- a/mirgecom/integrators/explicit_rk.py +++ b/mirgecom/integrators/explicit_rk.py @@ -28,17 +28,11 @@ """ -def rk4_step(state, t, dt, rhs, reference_state=None): +def rk4_step(state, t, dt, rhs): """Take one step using the fourth-order Classical Runge-Kutta method.""" - if reference_state is not None: - k1 = rhs(t, state, reference_state) - k2 = rhs(t+dt/2, state + dt/2*k1, reference_state) - k3 = rhs(t+dt/2, state + dt/2*k2, reference_state) - k4 = rhs(t+dt, state + dt*k3, reference_state) - else: - k1 = rhs(t, state) - k2 = rhs(t+dt/2, state + dt/2*k1) - k3 = rhs(t+dt/2, state + dt/2*k2) - k4 = rhs(t+dt, state + dt*k3) + k1 = rhs(t, state) + k2 = rhs(t+dt/2, state + dt/2*k1) + k3 = rhs(t+dt/2, state + dt/2*k2) + k4 = rhs(t+dt, state + dt*k3) return state + dt/6*(k1 + 2*k2 + 2*k3 + k4) diff --git a/mirgecom/integrators/lsrk.py b/mirgecom/integrators/lsrk.py index d50a9105a..b27dbae58 100644 --- a/mirgecom/integrators/lsrk.py +++ b/mirgecom/integrators/lsrk.py @@ -46,19 +46,12 @@ class LSRKCoefficients: C: np.ndarray -def lsrk_step(coefs, state, t, dt, rhs, reference_state=None): +def lsrk_step(coefs, state, t, dt, rhs): """Take one step using a low-storage Runge-Kutta method.""" k = 0.0 * state - if reference_state is not None: - reference_state = state - for i in range(len(coefs.A)): - k = coefs.A[i]*k + dt*rhs(t + coefs.C[i]*dt, state, reference_state) - state += coefs.B[i]*k - else: - for i in range(len(coefs.A)): - k = coefs.A[i]*k + dt*rhs(t + coefs.C[i]*dt, state) - state += coefs.B[i]*k - + for i in range(len(coefs.A)): + k = coefs.A[i]*k + dt*rhs(t + coefs.C[i]*dt, state) + state += coefs.B[i]*k return state @@ -68,9 +61,9 @@ def lsrk_step(coefs, state, t, dt, rhs, reference_state=None): C=np.array([0.])) -def euler_step(state, t, dt, rhs, reference_state=None): +def euler_step(state, t, dt, rhs): """Take one step using the explicit, 1st-order accurate, Euler method.""" - return lsrk_step(EulerCoefs, state, t, dt, rhs, reference_state) + return lsrk_step(EulerCoefs, state, t, dt, rhs) LSRK54CarpenterKennedyCoefs = LSRKCoefficients( @@ -94,12 +87,12 @@ def euler_step(state, t, dt, rhs, reference_state=None): 2802321613138/2924317926251])) -def lsrk54_step(state, t, dt, rhs, reference_state=None): +def lsrk54_step(state, t, dt, rhs): """Take one step using an explicit 5-stage, 4th-order, LSRK method. Coefficients are summarized in [Hesthaven_2008]_, Section 3.4. """ - return lsrk_step(LSRK54CarpenterKennedyCoefs, state, t, dt, rhs, reference_state) + return lsrk_step(LSRK54CarpenterKennedyCoefs, state, t, dt, rhs) LSRK144NiegemannDiehlBuschCoefs = LSRKCoefficients( @@ -150,12 +143,11 @@ def lsrk54_step(state, t, dt, rhs, reference_state=None): 0.8734213127600976])) -def lsrk144_step(state, t, dt, rhs, reference_state=None): +def lsrk144_step(state, t, dt, rhs): """Take one step using an explicit 14-stage, 4th-order, LSRK method. This method is derived by Niegemann, Diehl, and Busch (2012), with an optimal stability region for advection-dominated flows. The LSRK coefficients are summarized in [Niegemann_2012]_, Table 3. """ - return lsrk_step(LSRK144NiegemannDiehlBuschCoefs, state, t, dt, rhs, - reference_state) + return lsrk_step(LSRK144NiegemannDiehlBuschCoefs, state, t, dt, rhs) diff --git a/mirgecom/steppers.py b/mirgecom/steppers.py index 2ea10c478..da921c873 100644 --- a/mirgecom/steppers.py +++ b/mirgecom/steppers.py @@ -75,8 +75,7 @@ def _advance_state_stepper_func(rhs, timestepper, t=0.0, istep=0, pre_step_callback=None, post_step_callback=None, - logmgr=None, eos=None, dim=None, - reference_state=None): + logmgr=None, eos=None, dim=None): """Advance state from some time (t) to some time (t_final). Parameters @@ -133,21 +132,15 @@ def _advance_state_stepper_func(rhs, timestepper, logmgr.tick_before() if pre_step_callback is not None: - state, dt = pre_step_callback(state=state, step=istep, t=t, dt=dt, - reference_state=reference_state) + state, dt = pre_step_callback(state=state, step=istep, t=t, dt=dt) - if reference_state is not None: - reference_state = state - - state = timestepper(state=state, t=t, dt=dt, rhs=compiled_rhs, - reference_state=reference_state) + state = timestepper(state=state, t=t, dt=dt, rhs=compiled_rhs) t += dt istep += 1 if post_step_callback is not None: - state, dt = post_step_callback(state=state, step=istep, t=t, dt=dt, - reference_state=reference_state) + state, dt = post_step_callback(state=state, step=istep, t=t, dt=dt) if logmgr: set_dt(logmgr, dt) @@ -283,8 +276,7 @@ def advance_state(rhs, timestepper, state, t_final, t=0.0, istep=0, dt=0, pre_step_callback=None, post_step_callback=None, - logmgr=None, eos=None, dim=None, - reference_state=None): + logmgr=None, eos=None, dim=None): """Determine what stepper we're using and advance the state from (t) to (t_final). Parameters @@ -367,8 +359,7 @@ def advance_state(rhs, timestepper, state, t_final, state=state, t=t, t_final=t_final, dt=dt, pre_step_callback=pre_step_callback, post_step_callback=post_step_callback, - istep=istep, logmgr=logmgr, eos=eos, dim=dim, - reference_state=reference_state + istep=istep, logmgr=logmgr, eos=eos, dim=dim ) return current_step, current_t, current_state From 7ac8b17b5961209efc6ed0db21f80e1739cf5458 Mon Sep 17 00:00:00 2001 From: CI Runner Date: Thu, 4 Nov 2021 13:47:48 -0500 Subject: [PATCH 375/385] Use multistate in driver. --- examples/autoignition-mpi.py | 65 +++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/examples/autoignition-mpi.py b/examples/autoignition-mpi.py index ccb45f72b..85a49ea7b 100644 --- a/examples/autoignition-mpi.py +++ b/examples/autoignition-mpi.py @@ -325,8 +325,9 @@ def get_temperature_update(state, temperature): # and instead of writing the *current* running temperature to the restart file, # we could write the *temperature_seed*. That could fix up the non-deterministic # restart issue. - temperature_seed = eos.temperature(current_state, - temperature_seed=temperature_seed) + current_dv = compute_dependent_vars(current_state, + temperature_seed=temperature_seed) + temperature_seed = current_dv.temperature # import ipdb # ipdb.set_trace() @@ -501,7 +502,8 @@ def my_get_timestep(t, dt, state): comm=comm, op=MPI.MAX) return ts_field, cfl, min(t_remaining, dt) - def my_pre_step(step, t, dt, state, reference_state): + def my_pre_step(step, t, dt, state): + cv = state[0] try: dv = None @@ -516,28 +518,28 @@ def my_pre_step(step, t, dt, state, reference_state): if do_health: if dv is None: - dv = compute_dependent_vars(state) - health_errors = global_reduce(my_health_check(state, dv), op="lor") + dv = compute_dependent_vars(cv) + health_errors = global_reduce(my_health_check(cv, dv), op="lor") if health_errors: if rank == 0: logger.info("Fluid solution failed health check.") raise MyRuntimeError("Failed simulation health check.") - ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=state) + ts_field, cfl, dt = my_get_timestep(t=t, dt=dt, state=cv) if do_status: if dv is None: - dv = compute_dependent_vars(state) + dv = compute_dependent_vars(cv) my_write_status(dt=dt, cfl=cfl, dv=dv) if do_restart: - my_write_restart(step=step, t=t, state=state) + my_write_restart(step=step, t=t, state=cv) if do_viz: - production_rates, = compute_production_rates(state) + production_rates, = compute_production_rates(cv) if dv is None: - dv = compute_dependent_vars(state) - my_write_viz(step=step, t=t, dt=dt, state=state, dv=dv, + dv = compute_dependent_vars(cv) + my_write_viz(step=step, t=t, dt=dt, state=cv, dv=dv, production_rates=production_rates, ts_field=ts_field, cfl=cfl) @@ -550,27 +552,29 @@ def my_pre_step(step, t, dt, state, reference_state): return state, dt - def my_post_step(step, t, dt, state, reference_state): - ref_dv = compute_dependent_vars(reference_state) - new_dv = compute_dependent_vars(state, # noqa - temperature_seed=ref_dv.temperature) + def my_post_step(step, t, dt, state): + cv = state[0] + new_dv = compute_dependent_vars(cv, # noqa + temperature_seed=state[1]) + # Logmgr needs to know about EOS, dt, dim? # imo this is a design/scope flaw if logmgr: set_dt(logmgr, dt) - set_sim_state(logmgr, dim, state, eos) + set_sim_state(logmgr, dim, cv, eos) logmgr.tick_after() - return state, dt + return make_obj_array([cv, new_dv.temperature]), dt - def my_rhs(t, state, reference_state): - ref_dv = eos.dependent_vars(reference_state) - current_dv = eos.dependent_vars(state, - temperature_seed=ref_dv.temperature) + def my_rhs(t, state): + cv = state[0] + current_dv = eos.dependent_vars(cv, + temperature_seed=state[1]) - return (euler_operator(discr, cv=state, time=t, + return make_obj_array([euler_operator(discr, cv=cv, time=t, boundaries=boundaries, eos=eos, dv=current_dv) - + eos.get_species_source_terms(state)) + + eos.get_species_source_terms(cv), + 0*state[1]]) current_dt = get_sim_timestep(discr, current_state, current_t, current_dt, current_cfl, eos, t_final, constant_cfl) @@ -579,21 +583,22 @@ def my_rhs(t, state, reference_state): advance_state(rhs=my_rhs, timestepper=timestepper, pre_step_callback=my_pre_step, post_step_callback=my_post_step, dt=current_dt, - state=current_state, t=current_t, t_final=t_final, - reference_state=current_state) + state=make_obj_array([current_state, temperature_seed]), + t=current_t, t_final=t_final) # Dump the final data if rank == 0: logger.info("Checkpointing final state ...") - final_dv = compute_dependent_vars(current_state) - final_dm, = compute_production_rates(current_state) + final_cv = current_state[0] + final_dv = compute_dependent_vars(final_cv) + final_dm, = compute_production_rates(final_cv) ts_field, cfl, dt = my_get_timestep(t=current_t, dt=current_dt, - state=current_state) - my_write_viz(step=current_step, t=current_t, dt=dt, state=current_state, + state=final_cv) + my_write_viz(step=current_step, t=current_t, dt=dt, state=final_cv, dv=final_dv, production_rates=final_dm, ts_field=ts_field, cfl=cfl) my_write_status(dt=dt, cfl=cfl, dv=final_dv) - my_write_restart(step=current_step, t=current_t, state=current_state) + my_write_restart(step=current_step, t=current_t, state=final_cv) if logmgr: logmgr.close() From cd175b9ded03cc30b979420c25adaacfd84df263 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 4 Nov 2021 13:59:46 -0500 Subject: [PATCH 376/385] Update isolator production driver to main branch --- .ci-support/production-drivers-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-drivers-install.sh b/.ci-support/production-drivers-install.sh index f04273d88..9b84b8ae4 100755 --- a/.ci-support/production-drivers-install.sh +++ b/.ci-support/production-drivers-install.sh @@ -12,7 +12,7 @@ # The default values result in an install of the Y1 nozzle driver and # Wyatt Hagen's isolator driver that work with current MIRGE-Com # production branch: mirgecom@y1-production. -PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@y2-production:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar"} +PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@main:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar"} # Loop over the production drivers, clone them, and prepare for execution set -x OIFS="$IFS" From cf17ab41be2bfe82acc5c363fe65ef24d9c3563f Mon Sep 17 00:00:00 2001 From: CI Runner Date: Thu, 4 Nov 2021 14:02:17 -0500 Subject: [PATCH 377/385] Update inviscid flux tests to new interface --- test/test_inviscid.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/test_inviscid.py b/test/test_inviscid.py index c97b37501..55c9f3033 100644 --- a/test/test_inviscid.py +++ b/test/test_inviscid.py @@ -120,7 +120,8 @@ def rand(): # }}} - flux = inviscid_flux(discr, eos, cv) + pressure = eos.pressure(cv) + flux = inviscid_flux(discr, pressure, cv) flux_resid = flux - expected_flux for i in range(numeq, dim): @@ -174,7 +175,7 @@ def test_inviscid_flux_components(actx_factory, dim): energy = p_exact / 0.4 + 0.5 * np.dot(mom, mom) / mass cv = make_conserved(dim, mass=mass, energy=energy, momentum=mom) p = eos.pressure(cv) - flux = inviscid_flux(discr, eos, cv) + flux = inviscid_flux(discr, p, cv) def inf_norm(x): return actx.to_numpy(discr.norm(x, np.inf)) @@ -239,7 +240,7 @@ def inf_norm(x): return actx.to_numpy(discr.norm(x, np.inf)) assert inf_norm(p - p_exact) < tolerance - flux = inviscid_flux(discr, eos, cv) + flux = inviscid_flux(discr, p, cv) logger.info(f"{dim}d flux = {flux}") vel_exact = mom / mass From 6dce7bb438a47214bffe5645ebf9834542abb814 Mon Sep 17 00:00:00 2001 From: CI Runner Date: Thu, 4 Nov 2021 14:03:56 -0500 Subject: [PATCH 378/385] Switch to main branch for isolator production driver. --- .ci-support/production-drivers-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-drivers-install.sh b/.ci-support/production-drivers-install.sh index f04273d88..9b84b8ae4 100755 --- a/.ci-support/production-drivers-install.sh +++ b/.ci-support/production-drivers-install.sh @@ -12,7 +12,7 @@ # The default values result in an install of the Y1 nozzle driver and # Wyatt Hagen's isolator driver that work with current MIRGE-Com # production branch: mirgecom@y1-production. -PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@y2-production:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar"} +PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@main:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar"} # Loop over the production drivers, clone them, and prepare for execution set -x OIFS="$IFS" From ee7ec33c3371e1ced11e78f91d69855a98d7a42a Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 4 Nov 2021 15:46:48 -0500 Subject: [PATCH 379/385] Update isolator production driver to main branch (#548) --- .ci-support/production-drivers-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci-support/production-drivers-install.sh b/.ci-support/production-drivers-install.sh index f04273d88..9b84b8ae4 100755 --- a/.ci-support/production-drivers-install.sh +++ b/.ci-support/production-drivers-install.sh @@ -12,7 +12,7 @@ # The default values result in an install of the Y1 nozzle driver and # Wyatt Hagen's isolator driver that work with current MIRGE-Com # production branch: mirgecom@y1-production. -PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@y2-production:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar"} +PRODUCTION_DRIVERS=${PRODUCTION_DRIVERS:-"illinois-ceesd/drivers_y1-nozzle@parallel-lazy:illinois-ceesd/drivers_y2-isolator@main:illinois-ceesd/drivers_flame1d@nodal-reduction-device-scalar"} # Loop over the production drivers, clone them, and prepare for execution set -x OIFS="$IFS" From 49ecc9d19331560827bd95d525d8a68b96cfd821 Mon Sep 17 00:00:00 2001 From: CI Runner Date: Thu, 4 Nov 2021 16:43:27 -0500 Subject: [PATCH 380/385] Update NS to new inviscid flux interface. --- mirgecom/navierstokes.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mirgecom/navierstokes.py b/mirgecom/navierstokes.py index 580a91666..24cd76f79 100644 --- a/mirgecom/navierstokes.py +++ b/mirgecom/navierstokes.py @@ -79,7 +79,8 @@ from meshmode.dof_array import thaw -def ns_operator(discr, eos, boundaries, cv, t=0.0): +def ns_operator(discr, eos, boundaries, cv, t=0.0, + dv=None): r"""Compute RHS of the Navier-Stokes equations. Returns @@ -116,6 +117,8 @@ def ns_operator(discr, eos, boundaries, cv, t=0.0): """ dim = discr.dim actx = cv.array_context + if dv is None: + dv = eos.dependent_vars(cv) def _elbnd_flux(discr, compute_interior_flux, compute_boundary_flux, int_tpair, xrank_pairs, boundaries): @@ -154,7 +157,7 @@ def t_grad_flux_bnd(btag): return boundaries[btag].t_gradient_flux(discr, btag=btag, cv=cv, eos=eos, time=t) - gas_t = eos.temperature(cv) + gas_t = dv.temperature t_int_tpair = TracePair("int_faces", interior=eos.temperature(cv_int_tpair.int), exterior=eos.temperature(cv_int_tpair.ext)) @@ -208,7 +211,7 @@ def fvisc_divergence_flux_boundary(btag): vol_term = ( viscous_flux(discr, eos=eos, cv=cv, grad_cv=grad_cv, grad_t=grad_t) - - inviscid_flux(discr, eos=eos, cv=cv) + - inviscid_flux(discr, pressure=dv.pressure, cv=cv) ).join() bnd_term = ( From 18591a11c35c2a4fb5afef1b1a52f5b03b257646 Mon Sep 17 00:00:00 2001 From: CI Runner Date: Thu, 4 Nov 2021 19:48:21 -0500 Subject: [PATCH 381/385] Update inviscid divergence flux boundary interface. --- mirgecom/boundary.py | 16 ++++++++-------- mirgecom/euler.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index dbba0be15..7049098f1 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -51,23 +51,23 @@ class FluidBoundary(metaclass=ABCMeta): r"""Abstract interface to fluid boundary treatment. - .. automethod:: inviscid_boundary_flux + .. automethod:: inviscid_divergence_flux """ @abstractmethod - def inviscid_boundary_flux(self, discr, btag, cv, eos, **kwargs): + def inviscid_divergence_flux(self, discr, btag, cv, eos, **kwargs): """Get the inviscid flux across the boundary faces.""" class FluidBC(FluidBoundary): r"""Abstract interface to boundary conditions. - .. automethod:: inviscid_boundary_flux + .. automethod:: inviscid_divergence_flux .. automethod:: boundary_pair """ @abstractmethod - def inviscid_boundary_flux(self, discr, btag, cv, eos, **kwargs): + def inviscid_divergence_flux(self, discr, btag, cv, eos, **kwargs): """Get the inviscid part of the physical flux across the boundary *btag*.""" @abstractmethod @@ -80,15 +80,15 @@ class PrescribedInviscidBoundary(FluidBC): .. automethod:: __init__ .. automethod:: boundary_pair - .. automethod:: inviscid_boundary_flux + .. automethod:: inviscid_divergence_flux """ - def __init__(self, inviscid_boundary_flux_func=None, boundary_pair_func=None, + def __init__(self, inviscid_divergence_flux_func=None, boundary_pair_func=None, inviscid_facial_flux_func=None, fluid_solution_func=None, fluid_solution_flux_func=None): """Initialize the PrescribedInviscidBoundary and methods.""" self._bnd_pair_func = boundary_pair_func - self._inviscid_bnd_flux_func = inviscid_boundary_flux_func + self._inviscid_bnd_flux_func = inviscid_divergence_flux_func self._inviscid_facial_flux_func = inviscid_facial_flux_func if not self._inviscid_facial_flux_func: self._inviscid_facial_flux_func = inviscid_facial_flux @@ -109,7 +109,7 @@ def boundary_pair(self, discr, btag, cv, **kwargs): ext_soln = self._fluid_soln_func(nodes, cv=int_soln, normal=nhat, **kwargs) return TracePair(btag, interior=int_soln, exterior=ext_soln) - def inviscid_boundary_flux(self, discr, btag, cv, eos, **kwargs): + def inviscid_divergence_flux(self, discr, btag, cv, eos, **kwargs): """Get the inviscid flux across the boundary faces.""" if self._inviscid_bnd_flux_func: actx = cv.array_context diff --git a/mirgecom/euler.py b/mirgecom/euler.py index 26886c259..b39675042 100644 --- a/mirgecom/euler.py +++ b/mirgecom/euler.py @@ -108,8 +108,8 @@ def euler_operator(discr, eos, boundaries, cv, time=0.0): part_tpair.dd, interior=make_conserved(discr.dim, q=part_tpair.int), exterior=make_conserved(discr.dim, q=part_tpair.ext))) for part_tpair in cross_rank_trace_pairs(discr, cv.join())) - + sum(boundaries[btag].inviscid_boundary_flux(discr, btag=btag, cv=cv, - eos=eos, time=time) + + sum(boundaries[btag].inviscid_divergence_flux(discr, btag=btag, cv=cv, + eos=eos, time=time) for btag in boundaries) ) q = -div_operator(discr, inviscid_flux_vol.join(), inviscid_flux_bnd.join()) From 97013bba00cc1680a97b9ca86186799f98b383e0 Mon Sep 17 00:00:00 2001 From: CI Runner Date: Thu, 4 Nov 2021 19:54:55 -0500 Subject: [PATCH 382/385] Sharpen docstrings --- mirgecom/boundary.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mirgecom/boundary.py b/mirgecom/boundary.py index 7049098f1..a4088fa2a 100644 --- a/mirgecom/boundary.py +++ b/mirgecom/boundary.py @@ -56,7 +56,7 @@ class FluidBoundary(metaclass=ABCMeta): @abstractmethod def inviscid_divergence_flux(self, discr, btag, cv, eos, **kwargs): - """Get the inviscid flux across the boundary faces.""" + """Get the inviscid boundary flux for the divergence operator.""" class FluidBC(FluidBoundary): @@ -68,7 +68,7 @@ class FluidBC(FluidBoundary): @abstractmethod def inviscid_divergence_flux(self, discr, btag, cv, eos, **kwargs): - """Get the inviscid part of the physical flux across the boundary *btag*.""" + """Get the inviscid boundary flux for the divergence operator.""" @abstractmethod def boundary_pair(self, discr, btag, cv, eos, **kwargs): @@ -110,7 +110,7 @@ def boundary_pair(self, discr, btag, cv, **kwargs): return TracePair(btag, interior=int_soln, exterior=ext_soln) def inviscid_divergence_flux(self, discr, btag, cv, eos, **kwargs): - """Get the inviscid flux across the boundary faces.""" + """Get the inviscid boundary flux for the divergence operator.""" if self._inviscid_bnd_flux_func: actx = cv.array_context boundary_discr = discr.discr_from_dd(btag) From e06aafb54777f2370e693dec5aea6130cd08029b Mon Sep 17 00:00:00 2001 From: CI Runner Date: Thu, 4 Nov 2021 20:18:17 -0500 Subject: [PATCH 383/385] Correct interface error --- mirgecom/eos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/eos.py b/mirgecom/eos.py index f189383b3..75a295ad3 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -92,7 +92,7 @@ def pressure(self, cv: ConservedVars): @abstractmethod def temperature(self, cv: ConservedVars, - reference_state: ConservedVars = None): + temperature_seed: DOFArray = None): """Get the gas temperature.""" @abstractmethod From a2787bc252fdce87ed80c624ecb0334ef52ca60e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kl=C3=B6ckner?= Date: Fri, 5 Nov 2021 09:09:24 -0500 Subject: [PATCH 384/385] Remove extraneous indentation in PR dev docs --- doc/development/pullrequests.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/development/pullrequests.rst b/doc/development/pullrequests.rst index c670d139f..d994c1be9 100644 --- a/doc/development/pullrequests.rst +++ b/doc/development/pullrequests.rst @@ -286,11 +286,11 @@ with this process. If the PR is accepted for merging to main, then mirgecom developers will update the production capabilities to be compatible with the changes before merging. - .. important:: + .. important:: - Any production environment customizations must be backed out before - merging the PR development to main. Never merge a PR development with - production environment customizations in-place. + Any production environment customizations must be backed out before + merging the PR development to main. Never merge a PR development with + production environment customizations in-place. Merging a pull request ---------------------- From 5b78cb250c523f1f457e53fc7dd983cfe4df1a63 Mon Sep 17 00:00:00 2001 From: CI Runner Date: Fri, 5 Nov 2021 11:37:59 -0500 Subject: [PATCH 385/385] Update from downstream --- test/test_navierstokes.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/test/test_navierstokes.py b/test/test_navierstokes.py index 10ef64829..45d52897f 100644 --- a/test/test_navierstokes.py +++ b/test/test_navierstokes.py @@ -45,7 +45,7 @@ from mirgecom.boundary import ( DummyBoundary, PrescribedViscousBoundary, - IsothermalNoSlipBoundary + AdiabaticNoslipMovingBoundary ) from mirgecom.eos import IdealSingleGas from mirgecom.transport import SimpleTransport @@ -138,14 +138,14 @@ def test_uniform_rhs(actx_factory, nspecies, dim, order): f"rhoy_rhs = {rhoy_rhs}\n" ) - assert discr.norm(rho_resid, np.inf) < tolerance - assert discr.norm(rhoe_resid, np.inf) < tolerance + assert actx.to_numpy(discr.norm(rho_resid, np.inf)) < tolerance + assert actx.to_numpy(discr.norm(rhoe_resid, np.inf)) < tolerance for i in range(dim): - assert discr.norm(mom_resid[i], np.inf) < tolerance + assert actx.to_numpy(discr.norm(mom_resid[i], np.inf)) < tolerance for i in range(nspecies): - assert discr.norm(rhoy_resid[i], np.inf) < tolerance + assert actx.to_numpy(discr.norm(rhoy_resid[i], np.inf)) < tolerance - err_max = discr.norm(rho_resid, np.inf) + err_max = actx.to_numpy(discr.norm(rho_resid, np.inf)) eoc_rec0.add_data_point(1.0 / nel_1d, err_max) # set a non-zero, but uniform velocity component @@ -165,15 +165,15 @@ def test_uniform_rhs(actx_factory, nspecies, dim, order): mom_resid = rhs_resid.momentum rhoy_resid = rhs_resid.species_mass - assert discr.norm(rho_resid, np.inf) < tolerance - assert discr.norm(rhoe_resid, np.inf) < tolerance + assert actx.to_numpy(discr.norm(rho_resid, np.inf)) < tolerance + assert actx.to_numpy(discr.norm(rhoe_resid, np.inf)) < tolerance for i in range(dim): - assert discr.norm(mom_resid[i], np.inf) < tolerance + assert actx.to_numpy(discr.norm(mom_resid[i], np.inf)) < tolerance for i in range(nspecies): - assert discr.norm(rhoy_resid[i], np.inf) < tolerance + assert actx.to_numpy(discr.norm(rhoy_resid[i], np.inf)) < tolerance - err_max = discr.norm(rho_resid, np.inf) + err_max = actx.to_numpy(discr.norm(rho_resid, np.inf)) eoc_rec1.add_data_point(1.0 / nel_1d, err_max) logger.info( @@ -286,8 +286,8 @@ def poiseuille_2d(x_vec, eos, cv=None, **kwargs): boundaries = { DTAG_BOUNDARY("-1"): PrescribedViscousBoundary(q_func=initializer), DTAG_BOUNDARY("+1"): PrescribedViscousBoundary(q_func=initializer), - DTAG_BOUNDARY("-2"): IsothermalNoSlipBoundary(), - DTAG_BOUNDARY("+2"): IsothermalNoSlipBoundary()} + DTAG_BOUNDARY("-2"): AdiabaticNoslipMovingBoundary(), + DTAG_BOUNDARY("+2"): AdiabaticNoslipMovingBoundary()} ns_rhs = ns_operator(discr, eos=eos, boundaries=boundaries, cv=cv_input, t=0.0) @@ -310,14 +310,15 @@ def poiseuille_2d(x_vec, eos, cv=None, **kwargs): ) tol_fudge = 2e-4 - assert discr.norm(rho_resid, np.inf) < tolerance - # assert discr.norm(rhoe_resid, np.inf) < tolerance - mom_err = [discr.norm(mom_resid[i], np.inf) for i in range(dim)] + assert actx.to_numpy(discr.norm(rho_resid, np.inf)) < tolerance + # assert actx.to_numpy(discr.norm(rhoe_resid, np.inf)) < tolerance + mom_err = [actx.to_numpy(discr.norm(mom_resid[i], np.inf)) + for i in range(dim)] err_max = max(mom_err) for i in range(dim): assert mom_err[i] < tol_fudge - # err_max = discr.norm(rho_resid, np.inf) + # err_max = actx.to_numpy(discr.norm(rho_resid, np.inf) eoc_rec.add_data_point(1.0 / nfac, err_max) logger.info(