Error while runing simulation

Hi everyone,

I’m trying to run a simulation on the GPU cluster, and I’m getting this error:

Traceback (most recent call last):
File “/usr/local/bin/pyfr”, line 33, in
sys.exit(load_entry_point(‘pyfr==2.0.1’, ‘console_scripts’, ‘pyfr’)())
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/main.py”, line 124, in main
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/main.py”, line 258, in process_run
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/main.py”, line 254, in _process_common
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/integrators/base.py”, line 141, in run
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/integrators/dual/phys/controllers.py”, line 43, in advance_to
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/integrators/dual/phys/steppers.py”, line 28, in step
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/integrators/dual/pseudo/multip.py”, line 304, in pseudo_advance
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/integrators/dual/pseudo/pseudocontrollers.py”, line 153, in pseudo_advance
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/integrators/dual/pseudo/pseudosteppers.py”, line 244, in step
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/integrators/dual/pseudo/multip.py”, line 102, in _rhs_with_dts
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/integrators/dual/pseudo/pseudosteppers.py”, line 37, in _rhs_with_dts
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/solvers/base/system.py”, line 270, in rhs
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/util.py”, line 38, in newmeth
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/solvers/baseadvecdiff/system.py”, line 132, in _rhs_graphs
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/solvers/base/system.py”, line 334, in _group
File “/usr/local/lib/python3.10/dist-packages/pyfr-2.0.1-py3.10.egg/pyfr/backends/base/types.py”, line 388, in group
ValueError: Kernel grouping violates dependencies
Traceback (most recent call last):
File “/usr/local/bin/pyfr”, line 33, in

MPI_ABORT was invoked on rank 4 in communicator MPI_COMM_WORLD
with errorcode 1.

NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes.
You may or may not see output from other processes, depending on
exactly when Open MPI kills them.

with the following ini file content:
[backend]
precision = double
rank-allocator = linear

[backend-cuda]
device-id = local-rank
;mpi-type = cuda-aware
mpi-type = standard

[constants]
ac-zeta = 0.4
gamma = 1.4
Uin = 1
Vin = 0
Win = 0
Pc = 1.013
Pr = 0.72
nu = 0.000015
h = 0.1
k = 0.41
Utau = 0.93
d = 0.063
z0 = 0.01

[solver]
system = ac-navier-stokes
order = 3
anti-alias = flux, surf-flux

[solver-time-integrator]
formulation = dual
scheme = backward-euler
pseudo-scheme = rk45
controller = none
tstart = 0.0
tend = 24
dt = 0.00016
atol = 0.00001
rtol = 0.00001
pseudo-dt = 1e-5
pseudo-niters-max = 20
pseudo-niters-min = 3
pseudo-resid-tol = 1e-4
; pseudo-resid-tol-p = 3e-3
pseudo-resid-norm = l2
pseudo-controller = local-pi
safety-fact = 0.8
min-fact = 0.98
max-fact = 1.01
pseudo-dt-max-mult = 2.5

[solver-dual-time-integrator-multip]
pseudo-dt-fact = 1.75
; cycle = [(4,1), (3, 1), (2, 1), (1, 1), (0, 3), (1, 1), (2, 1), (3, 1), (4,3)]
cycle = [(3, 1), (2, 1), (1, 1), (0, 3), (1, 1), (2, 1), (3, 3)]

[solver-interfaces]
riemann-solver = rusanov
ldg-beta = 0.5
ldg-tau = 0.1

[solver-interfaces-line]
flux-pts = gauss-legendre
quad-deg = 9
quad-pts = gauss-legendre
;[solver-interfaces-line-mg-p2]
;flux-pts = gauss-legendre
;quad-deg = 7
;quad-pts = gauss-legendre
;[solver-interfaces-line-mg-p1]
;flux-pts = gauss-legendre
;quad-deg = 5
;quad-pts = gauss-legendre
;[solver-interfaces-line-mg-p0]
;flux-pts = gauss-legendre
;quad-deg = 3
;quad-pts = gauss-legendre

[solver-elements-tri]
soln-pts = williams-shunn
quad-deg = 9
quad-pts = williams-shunn
;[solver-elements-tri-mg-p2]
;soln-pts = williams-shunn
;quad-deg = 7
;quad-pts = williams-shunn
;[solver-elements-tri-mg-p1]
;soln-pts = williams-shunn
;quad-deg = 5
;quad-pts = williams-shunn
;[solver-elements-tri-mg-p0]
;soln-pts = williams-shunn
;quad-deg = 3
;quad-pts = williams-shunn

[solver-elements-quad]
soln-pts = gauss-legendre
quad-deg = 9
quad-pts = gauss-legendre
;[solver-elements-quad-mg-p2]
;soln-pts = gauss-legendre
;quad-deg = 7
;quad-pts = gauss-legendre
;[solver-elements-quad-mg-p1]
;soln-pts = gauss-legendre
;quad-deg = 5
;quad-pts = gauss-legendre
;[solver-elements-quad-mg-p0]
;soln-pts = gauss-legendre
;quad-deg = 3
;quad-pts = gauss-legendre

[solver-elements-hex]
soln-pts = gauss-legendre
quad-deg = 9
quad-pts = gauss-legendre
;[solver-elements-hex-mg-p2]
;soln-pts = gauss-legendre
;quad-deg = 7
;quad-pts = gauss-legendre
;[solver-elements-hex-mg-p1]
;soln-pts = gauss-legendre
;quad-deg = 5
;quad-pts = gauss-legendre
;[solver-elements-hex-mg-p0]
;soln-pts = gauss-legendre
;quad-deg = 3
;quad-pts = gauss-legendre

[solver-elements-pri]
soln-pts = williams-shunn~gauss-legendre
quad-deg = 9
quad-pts = williams-shunn~gauss-legendre
;[solver-elements-pri-mg-p2]
;soln-pts = williams-shunn~gauss-legendre
;quad-deg = 7
;quad-pts = williams-shunn~gauss-legendre
;[solver-elements-pri-mg-p1]
;soln-pts = williams-shunn~gauss-legendre
;quad-deg = 5
;quad-pts = williams-shunn~gauss-legendre
;[solver-elements-pri-mg-p0]
;soln-pts = williams-shunn~gauss-legendre
;quad-deg = 3
;quad-pts = williams-shunn~gauss-legendre

[solver-interfaces-tri]
flux-pts = williams-shunn
quad-deg = 9
quad-pts = williams-shunn
;[solver-interfaces-tri-mg-p2]
;flux-pts = williams-shunn
;quad-deg = 7
;quad-pts = williams-shunn
;[solver-interfaces-tri-mg-p1]
;flux-pts = williams-shunn
;quad-deg = 5
;quad-pts = williams-shunn
;[solver-interfaces-tri-mg-p0]
;flux-pts = williams-shunn
;quad-deg = 3
;quad-pts = williams-shunn

[solver-interfaces-quad]
flux-pts = gauss-legendre
quad-deg = 9
quad-pts = gauss-legendre
;[solver-interfaces-quad-mg-p2]
;flux-pts = gauss-legendre
;quad-deg = 7
;quad-pts = gauss-legendre
;[solver-interfaces-quad-mg-p1]
;flux-pts = gauss-legendre
;quad-deg = 5
;quad-pts = gauss-legendre
;[solver-interfaces-quad-mg-p0]
;flux-pts = gauss-legendre
;quad-deg = 3
;quad-pts = gauss-legendre

[soln-plugin-nancheck]
nsteps = 50

[soln-plugin-writer]
tstart = 12.0
dt-out = 0.5
basedir = /home/amir.shaulov/
; basedir = .
basename = Urban_AC-{t:.2f}

[solver-plugin-turbulence]
avg-rho = 1.185
avg-u = 1.0
avg-mach = 0.02
turbulence-intensity = 8
turbulence-length-scale = 0.002
sigma = 0.7
centre = (-0.4, 0.0, 0.06)
y-dim = 0.8
z-dim = 0.12
rot-axis = (0, 0, 1)
rot-angle = 0.0

[soln-ics]
rho=1.185
u = 1.0
v = 0.0
w = 0.0
p = Pc

[soln-bcs-walls]
type = no-slp-wall
u = 0
v = 0
w = 0

[soln-bcs-Inlet]
type = ac-char-riem-inv
ac-zeta = 5
rho=1.185
p = Pc
u = (Utau/k) * 2.303*log((z-d)/(z0))
v = Vin
w = Win

[soln-bcs-inlet]
type = ac-char-riem-inv
ac-zeta = 5
rho=1.185
p = Pc
u = (Utau/k) * 2.303*log((z-d)/(z0))
v = Vin
w = Win

[soln-bcs-Outlet]
type = ac-char-riem-inv
ac-zeta = 5
rho=1.185
p = Pc
u = Uin
v = Vin
w = Win

[soln-bcs-outlet]
type = ac-char-riem-inv
ac-zeta = 5
rho=1.185
p = Pc
u = Uin
v = Vin
w = Win

[soln-bcs-farfield]
type = slp-wall
ac-zeta = 5
rho=1.185
p = Pc

Please advice how to overcome this issue.

Amir

I think updating to v2.0.3 should fix the issue.

Regards

Peter