PyFR 1.3.0: OpenMP backend error

Hello, I’ve been using PyFR on a windows laptop with Cygwin 64bit, and after upgrading pyfr to 1.3, I get an error which seems to persist even when I downgrade back to 1.2.

$ pyfr run -b openmp -p euler_vortex_2d.pyfrm euler_vortex_2d.ini

Traceback (most recent call last):
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/util.py", line 32, in __call__
KeyError: (<function OpenMPKernelProvider._build_kernel at 0x6fffda28c80>, b'(X\x08\x00\x00\x00par_gemmq\x00X\x96\n\x00\x00\n\n#include <omp.h>\n#include <stdlib.h>\n#include <tgmath.h>\n\n#define PYFR_ALIGN_BYTES 32\n#define PYFR_NOINLINE __attribute__ ((noinline))\n\n#define min(a, b) ((a) < (b) ? (a) : (b))\n#define max(a, b) ((a) > (b) ? (a) : (b))\n\n// Typedefs\ntypedef double fpdtype_t;\n\n// OpenMP static loop scheduling functions\n\nstatic inline int\ngcd(int a, int b)\n{\n return (a == 0) ? b : gcd(b % a, a);\n}\n\nstatic inline void\nloop_sched_1d(int n, int align, int *b, int *e)\n{\n int tid = omp_get_thread_num();\n int nth = omp_get_num_threads();\n\n // Round up n to be a multiple of nth\n int rn = n + nth - 1 - (n - 1) % nth;\n\n // Nominal tile size\n int sz = rn / nth;\n\n // Handle alignment\n sz += align - 1 - (sz - 1) % align;\n\n // Assign the starting and ending index\n *b = sz * tid;\n *e = min(*b + sz, n);\n\n // Clamp\n if (*b >= n)\n *b = *e = 0;\n}\n\nstatic inline void\nloop_sched_2d(int nrow, int ncol, int colalign,\n int *rowb, int *rowe, int *colb, int *cole)\n{\n int tid = omp_get_thread_num();\n int nth = omp_get_num_threads();\n\n // Distribute threads\n int nrowth = gcd(nrow, nth);\n int ncolth = nth / nrowth;\n\n // Row and column indices for our thread\n int rowix = tid / ncolth;\n int colix = tid % ncolth;\n\n // Round up ncol to be a multiple of ncolth\n int rncol = ncol + ncolth - 1 - (ncol - 1) % ncolth;\n\n // Nominal tile size\n int ntilerow = nrow / nrowth;\n int ntilecol = rncol / ncolth;\n\n // Handle column alignment\n ntilecol += colalign - 1 - (ntilecol - 1) % colalign;\n\n // Assign the starting and ending row to each thread\n *rowb = ntilerow * rowix;\n *rowe = *rowb + ntilerow;\n\n // Assign the starting and ending column to each thread\n *colb = ntilecol * colix;\n *cole = min(*colb + ntilecol, ncol);\n\n // Clamp\n if (*colb >= ncol)\n *colb = *cole = 0;\n}\n\n\n\n\n// CBLAS GEMM constants\n#define ROW_MAJOR 101\n#define NO_TRANS 111\n\n// CBLAS GEMM prototype\ntypedef void (*cblas_gemm_t)(int, int, int,\n int, int, int,\n fpdtype_t, const fpdtype_t *, int,\n const fpdtype_t *, int,\n fpdtype_t, fpdtype_t *, int);\n\nvoid\npar_gemm(cblas_gemm_t gemm, int M, int N, int K,\n fpdtype_t alpha, const fpdtype_t *A, int lda,\n const fpdtype_t *B, int ldb,\n fpdtype_t beta, fpdtype_t *C, int ldc)\n{\n #pragma omp parallel\n {\n int begin, end;\n loop_sched_1d(N, PYFR_ALIGN_BYTES / sizeof(fpdtype_t), &begin, &end);\n\n gemm(ROW_MAJOR, NO_TRANS, NO_TRANS, M, end - begin, K,\n alpha, A, lda, B + begin, ldb, beta, C + begin, ldc);\n }\n}\n\nq\x01]q\x02(cnumpy\nint64\nq\x03cnumpy\nint32\nq\x04h\x04h\x04cnumpy\nfloat64\nq\x05h\x03h\x04h\x03h\x04h\x05h\x03h\x04etq\x06.', b'}q\x00.')

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/User/PyFR/env/bin/pyfr", line 9, in <module>
load_entry_point('pyfr==1.2.0', 'console_scripts', 'pyfr')()
File "/usr/lib/python3.4/site-packages/mpmath-0.19-py3.4.egg/mpmath/ctx_mp.py", line 1301, in g
return f(*args, **kwargs)
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/scripts/main.py", line 112, in main
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/scripts/main.py", line 233, in process_run
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/scripts/main.py", line 217, in _process_common
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/solvers/__init__.py", line 14, in get_solver
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/integrators/__init__.py", line 26, in get_integrator
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/integrators/controllers.py", line 14, in __init__
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/integrators/steppers.py", line 9, in __init__
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/integrators/base.py", line 41, in __init__
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/solvers/base/system.py", line 59, in __init__
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/solvers/base/system.py", line 167, in _gen_kernels
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/solvers/baseadvec/elements.py", line 80, in <lambda>
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/backends/base/backend.py", line 173, in kernel
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/backends/openmp/cblas.py", line 94, in mul
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/util.py", line 34, in __call__
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/backends/openmp/provider.py", line 13, in _build_kernel
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/backends/openmp/compiler.py", line 61, in __init__
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/backends/openmp/compiler.py", line 30, in __init__
File "/home/User/PyFR/env/lib/python3.4/site-packages/pyfr-1.2.0-py3.4.egg/pyfr/backends/openmp/compiler.py", line 80, in _build
File "/home/User/PyFR/env/lib/python3.4/site-packages/pytools/prefork.py", line 197, in call_capture_output
return forker[0].call_capture_output(cmdline, cwd, error_on_nonzero)
File "/home/User/PyFR/env/lib/python3.4/site-packages/pytools/prefork.py", line 156, in call_capture_output
error_on_nonzero)
File "/home/User/PyFR/env/lib/python3.4/site-packages/pytools/prefork.py", line 136, in _remote_invoke
who="Prefork client", partner="prefork server")
File "/home/User/PyFR/env/lib/python3.4/site-packages/pytools/prefork.py", line 70, in _recv_packet
size_bytes = sock.recv(size_bytes_size)
ConnectionResetError: [Errno 104] Connection reset by peer
Error in atexit._run_exitfuncs:
Traceback (most recent call last):
File "/home/User/PyFR/env/lib/python3.4/site-packages/pytools/prefork.py", line 144, in _quit
self._remote_invoke("quit")
File "/home/User/PyFR/env/lib/python3.4/site-packages/pytools/prefork.py", line 134, in _remote_invoke
_send_packet(self.socket, (name, args, kwargs))
File "/home/User/PyFR/env/lib/python3.4/site-packages/pytools/prefork.py", line 64, in _send_packet
sock.sendall(pack("I", len(packet)))
ConnectionResetError: [Errno 104] Connection reset by peer

since it refers to prefork, I thought it would be an issue with pytools, but changing versions of that seemed to not have any affect.

It may be worth noting that for v1.1 and 1.2 I’ve downloaded the program and used setup.py install, but for 1.3 my mentor suggested using pip -e and then pip. after creating a few virtual environments, I’m not sure how to approach the problem.

Hi Leo,

Hello, I've been using PyFR on a windows laptop with Cygwin 64bit, and
after upgrading pyfr to 1.3, I get an error which seems to persist even
when I downgrade back to 1.2.

Nothing substantial changed between v1.2.0 and v1.3.0 with regards to
the OpenMP backend. It is extremely likely that this error is due to
either a new version of pytools or Cygwin.

Given you are running on a laptop your best bet is to comment out the
enable_prefork() line on:

https://github.com/vincentlab/PyFR/blob/develop/pyfr/scripts/main.py#L196

Regards, Freddie.

Hi Freddie,

I added the change, and it seemed to be a similar error being that it stopped earlier in the program.

Traceback (most recent call last):
File "/home/User/PyFR/env4/src/pyfr/pyfr/util.py", line 32, in __call__
res = cache[key]
KeyError: (<function OpenMPKernelProvider._build_kernel at 0x6fffe8ca6a8>, b'\x80\x03X\x08\x00\x00\x00par_gemmq\x00X\x96\n\x00\x00\n\n#include <omp.h>\n#include <stdlib.h>\n#include <tgmath.h>\n\n#define PYFR_ALIGN_BYTES 32\n#define PYFR_NOINLINE __attribute__ ((noinline))\n\n#define min(a, b) ((a) < (b) ? (a) : (b))\n#define max(a, b) ((a) > (b) ? (a) : (b))\n\n// Typedefs\ntypedef double fpdtype_t;\n\n// OpenMP static loop scheduling functions\n\nstatic inline int\ngcd(int a, int b)\n{\n return (a == 0) ? b : gcd(b % a, a);\n}\n\nstatic inline void\nloop_sched_1d(int n, int align, int *b, int *e)\n{\n int tid = omp_get_thread_num();\n int nth = omp_get_num_threads();\n\n // Round up n to be a multiple of nth\n int rn = n + nth - 1 - (n - 1) % nth;\n\n // Nominal tile size\n int sz = rn / nth;\n\n // Handle alignment\n sz += align - 1 - (sz - 1) % align;\n\n // Assign the starting and ending index\n *b = sz * tid;\n *e = min(*b + sz, n);\n\n // Clamp\n if (*b >= n)\n *b = *e = 0;\n}\n\nstatic inline void\nloop_sched_2d(int nrow, int ncol, int colalign,\n int *rowb, int *rowe, int *colb, int *cole)\n{\n int tid = omp_get_thread_num();\n int nth = omp_get_num_threads();\n\n // Distribute threads\n int nrowth = gcd(nrow, nth);\n int ncolth = nth / nrowth;\n\n // Row and column indices for our thread\n int rowix = tid / ncolth;\n int colix = tid % ncolth;\n\n // Round up ncol to be a multiple of ncolth\n int rncol = ncol + ncolth - 1 - (ncol - 1) % ncolth;\n\n // Nominal tile size\n int ntilerow = nrow / nrowth;\n int ntilecol = rncol / ncolth;\n\n // Handle column alignment\n ntilecol += colalign - 1 - (ntilecol - 1) % colalign;\n\n // Assign the starting and ending row to each thread\n *rowb = ntilerow * rowix;\n *rowe = *rowb + ntilerow;\n\n // Assign the starting and ending column to each thread\n *colb = ntilecol * colix;\n *cole = min(*colb + ntilecol, ncol);\n\n // Clamp\n if (*colb >= ncol)\n *colb = *cole = 0;\n}\n\n\n\n\n// CBLAS GEMM constants\n#define ROW_MAJOR 101\n#define NO_TRANS 111\n\n// CBLAS GEMM prototype\ntypedef void (*cblas_gemm_t)(int, int, int,\n int, int, int,\n fpdtype_t, const fpdtype_t *, int,\n const fpdtype_t *, int,\n fpdtype_t, fpdtype_t *, int);\n\nvoid\npar_gemm(cblas_gemm_t gemm, int M, int N, int K,\n fpdtype_t alpha, const fpdtype_t *A, int lda,\n const fpdtype_t *B, int ldb,\n fpdtype_t beta, fpdtype_t *C, int ldc)\n{\n #pragma omp parallel\n {\n int begin, end;\n loop_sched_1d(N, PYFR_ALIGN_BYTES / sizeof(fpdtype_t), &begin, &end);\n\n gemm(ROW_MAJOR, NO_TRANS, NO_TRANS, M, end - begin, K,\n alpha, A, lda, B + begin, ldb, beta, C + begin, ldc);\n }\n}\n\nq\x01]q\x02(cnumpy\nint64\nq\x03cnumpy\nint32\nq\x04h\x04h\x04cnumpy\nfloat64\nq\x05h\x03h\x04h\x03h\x04h\x05h\x03h\x04e\x87q\x06.', b'\x80\x03}q\x00.')

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/User/PyFR/env4/lib/python3.4/site-packages/pytools/prefork.py", line 50, in call_capture_output
% (popen.returncode, " ".join(cmdline), stderr_data))
pytools.prefork.ExecError: status 1 invoking 'gcc -shared -std=c99 -Ofast -march=native -fopenmp -fPIC -o libtmp.so tmp.c -mno-cygwin': b'gcc: error: unrecognized command line option \xe2\x80\x98-mno-cygwin\xe2\x80\x99\n'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/User/PyFR/env4/bin/pyfr", line 9, in <module>
load_entry_point('pyfr==1.3.0', 'console_scripts', 'pyfr')()
File "/home/User/PyFR/env4/src/pyfr/pyfr/scripts/main.py", line 109, in main
args.process(args)
File "/home/User/PyFR/env4/src/pyfr/pyfr/scripts/main.py", line 230, in process_run
args, NativeReader(args.mesh), None, Inifile.load(args.cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/scripts/main.py", line 214, in _process_common
solver = get_solver(backend, rallocs, mesh, soln, cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/solvers/__init__.py", line 14, in get_solver
return get_integrator(backend, systemcls, rallocs, mesh, initsoln, cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/integrators/__init__.py", line 26, in get_integrator
return integrator(backend, systemcls, rallocs, mesh, initsoln, cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/integrators/controllers.py", line 14, in __init__
super().__init__(*args, **kwargs)
File "/home/User/PyFR/env4/src/pyfr/pyfr/integrators/steppers.py", line 9, in __init__
super().__init__(*args, **kwargs)
File "/home/User/PyFR/env4/src/pyfr/pyfr/integrators/base.py", line 41, in __init__
self.system = systemcls(backend, rallocs, mesh, initsoln, nreg, cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/solvers/base/system.py", line 59, in __init__
self._gen_kernels(eles, int_inters, mpi_inters, bc_inters)
File "/home/User/PyFR/env4/src/pyfr/pyfr/solvers/base/system.py", line 167, in _gen_kernels
kernels[pn, kn].append(kgetter())
File "/home/User/PyFR/env4/src/pyfr/pyfr/solvers/baseadvec/elements.py", line 57, in <lambda>
out=self._scal_fpts
File "/home/User/PyFR/env4/src/pyfr/pyfr/backends/base/backend.py", line 154, in kernel
return kern(*args, **kwargs)
File "/home/User/PyFR/env4/src/pyfr/pyfr/backends/openmp/cblas.py", line 93, in mul
par_gemm = self._build_kernel('par_gemm', src, argt)
File "/home/User/PyFR/env4/src/pyfr/pyfr/util.py", line 34, in __call__
res = cache[key] = self.func(*args, **kwargs)
File "/home/User/PyFR/env4/src/pyfr/pyfr/backends/openmp/provider.py", line 13, in _build_kernel
mod = GccSourceModule(src, self.backend.cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/backends/openmp/compiler.py", line 61, in __init__
super().__init__(src, cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/backends/openmp/compiler.py", line 30, in __init__
lname = self._build(tmpdir)
File "/home/User/PyFR/env4/src/pyfr/pyfr/backends/openmp/compiler.py", line 80, in _build
call_capture_output(cmd + self._cflags, cwd=tmpdir)
File "/home/User/PyFR/env4/lib/python3.4/site-packages/pytools/prefork.py", line 197, in call_capture_output
return forker[0].call_capture_output(cmdline, cwd, error_on_nonzero)
File "/home/User/PyFR/env4/lib/python3.4/site-packages/pytools/prefork.py", line 54, in call_capture_output
% ( " ".join(cmdline), e))
pytools.prefork.ExecError: error invoking 'gcc -shared -std=c99 -Ofast -march=native -fopenmp -fPIC -o libtmp.so tmp.c -mno-cygwin': status 1 invoking 'gcc -shared -std=c99 -Ofast -march=native -fopenmp -fPIC -o libtmp.so tmp.c -mno-cygwin': b'gcc: error: unrecognized command line option \xe2\x80\x98-mno-cygwin\xe2\x80\x99\n'

My tutor suggested I did a rebaseall in ash before making the change, which I had done. It is also worth noting that whenever I try running pyfr in a new environment for the first time, microsoft security essentials asks whether I want to allow the program to run. I don’t know if it may be to do with the firewall, but is interesting nonetheless.

Regards, Leo

I realised that there was a cflag that was added into the inifile ‘-mno-cygwin’, so I just removed that. Here’s the new crash:

$ pyfr run -b openmp -p euler_vortex_2d.pyfrm euler_vortex_2d.ini

0 [main] python3 6872 child_info_fork::abort: unable to map C:\cygwin64\tmp\pyfr-0-jgliw1v5\libtmp.so, Win32 error 126
Traceback (most recent call last):
File "/home/User/PyFR/env4/src/pyfr/pyfr/util.py", line 32, in __call__
res = cache[key]
KeyError: (<function OpenMPKernelProvider._build_kernel at 0x6fffe18b6a8>, b'\x80\x03X\x05\x00\x00\x00tfluxq\x00X\xc5\x14\x00\x00\n\n#include <omp.h>\n#include <stdlib.h>\n#include <tgmath.h>\n\n#define PYFR_ALIGN_BYTES 32\n#define PYFR_NOINLINE __attribute__ ((noinline))\n\n#define min(a, b) ((a) < (b) ? (a) : (b))\n#define max(a, b) ((a) > (b) ? (a) : (b))\n\n// Typedefs\ntypedef double fpdtype_t;\n\n// OpenMP static loop scheduling functions\n\nstatic inline int\ngcd(int a, int b)\n{\n return (a == 0) ? b : gcd(b % a, a);\n}\n\nstatic inline void\nloop_sched_1d(int n, int align, int *b, int *e)\n{\n int tid = omp_get_thread_num();\n int nth = omp_get_num_threads();\n\n // Round up n to be a multiple of nth\n int rn = n + nth - 1 - (n - 1) % nth;\n\n // Nominal tile size\n int sz = rn / nth;\n\n // Handle alignment\n sz += align - 1 - (sz - 1) % align;\n\n // Assign the starting and ending index\n *b = sz * tid;\n *e = min(*b + sz, n);\n\n // Clamp\n if (*b >= n)\n *b = *e = 0;\n}\n\nstatic inline void\nloop_sched_2d(int nrow, int ncol, int colalign,\n int *rowb, int *rowe, int *colb, int *cole)\n{\n int tid = omp_get_thread_num();\n int nth = omp_get_num_threads();\n\n // Distribute threads\n int nrowth = gcd(nrow, nth);\n int ncolth = nth / nrowth;\n\n // Row and column indices for our thread\n int rowix = tid / ncolth;\n int colix = tid % ncolth;\n\n // Round up ncol to be a multiple of ncolth\n int rncol = ncol + ncolth - 1 - (ncol - 1) % ncolth;\n\n // Nominal tile size\n int ntilerow = nrow / nrowth;\n int ntilecol = rncol / ncolth;\n\n // Handle column alignment\n ntilecol += colalign - 1 - (ntilecol - 1) % colalign;\n\n // Assign the starting and ending row to each thread\n *rowb = ntilerow * rowix;\n *rowe = *rowb + ntilerow;\n\n // Assign the starting and ending column to each thread\n *colb = ntilecol * colix;\n *cole = min(*colb + ntilecol, ncol);\n\n // Clamp\n if (*colb >= ncol)\n *colb = *cole = 0;\n}\n\n\n\n\n\n\n\n\n\nstatic PYFR_NOINLINE void tflux_inner(int _nx, fpdtype_t *__restrict__ f_v0v0, fpdtype_t *__restrict__ f_v0v1, fpdtype_t *__restrict__ f_v0v2, fpdtype_t *__restrict__ f_v0v3, fpdtype_t *__restrict__ f_v1v0, fpdtype_t *__restrict__ f_v1v1, fpdtype_t *__restrict__ f_v1v2, fpdtype_t *__restrict__ f_v1v3, const fpdtype_t *__restrict__ smats_v0v0, const fpdtype_t *__restrict__ smats_v0v1, const fpdtype_t *__restrict__ smats_v1v0, const fpdtype_t *__restrict__ smats_v1v1, const fpdtype_t *__restrict__ u_v0, const fpdtype_t *__restrict__ u_v1, const fpdtype_t *__restrict__ u_v2, const fpdtype_t *__restrict__ u_v3)\n {\n for (int _x = 0; _x < _nx; _x++)\n {\n \n // Compute the flux\n fpdtype_t ftemp[2][4];\n fpdtype_t p, v[2];\n {\n\n fpdtype_t invrho_ = 1.0/u_v0[_x], E_ = u_v3[_x];\n\n // Compute the velocities\n fpdtype_t rhov_[2];\n rhov_[0] = u_v1[_x];\n v[0] = invrho_*rhov_[0];\n rhov_[1] = u_v2[_x];\n v[1] = invrho_*rhov_[1];\n\n // Compute the pressure\n p = 0.3999999999999999*(E_ - 0.5*invrho_*((rhov_[0])*(rhov_[0]) + (rhov_[1])*(rhov_[1])));\n\n // Density and energy fluxes\n ftemp[0][0] = rhov_[0];\n ftemp[0][3] = (E_ + p)*v[0];\n ftemp[1][0] = rhov_[1];\n ftemp[1][3] = (E_ + p)*v[1];\n\n // Momentum fluxes\n ftemp[0][1] = rhov_[0]*v[0] + p;\n ftemp[0][2] = rhov_[0]*v[1];\n ftemp[1][1] = rhov_[1]*v[0];\n ftemp[1][2] = rhov_[1]*v[1] + p;\n\n};\n\n // Transform the fluxes\n f_v0v0[_x] = smats_v0v0[_x]*ftemp[0][0] + smats_v0v1[_x]*ftemp[1][0];\n f_v0v1[_x] = smats_v0v0[_x]*ftemp[0][1] + smats_v0v1[_x]*ftemp[1][1];\n f_v0v2[_x] = smats_v0v0[_x]*ftemp[0][2] + smats_v0v1[_x]*ftemp[1][2];\n f_v0v3[_x] = smats_v0v0[_x]*ftemp[0][3] + smats_v0v1[_x]*ftemp[1][3];\n f_v1v0[_x] = smats_v1v0[_x]*ftemp[0][0] + smats_v1v1[_x]*ftemp[1][0];\n f_v1v1[_x] = smats_v1v0[_x]*ftemp[0][1] + smats_v1v1[_x]*ftemp[1][1];\n f_v1v2[_x] = smats_v1v0[_x]*ftemp[0][2] + smats_v1v1[_x]*ftemp[1][2];\n f_v1v3[_x] = smats_v1v0[_x]*ftemp[0][3] + smats_v1v1[_x]*ftemp[1][3];\n\n }\n }\n void tflux(int _ny, int _nx, fpdtype_t* __restrict__ f_v, int lsdf, const fpdtype_t* __restrict__ smats_v, int lsdsmats, const fpdtype_t* __restrict__ u_v, int lsdu)\n {\n #pragma omp parallel\n {\n int align = PYFR_ALIGN_BYTES / sizeof(fpdtype_t);\n int rb, re, cb, ce;\n loop_sched_2d(_ny, _nx, align, &rb, &re, &cb, &ce);\n for (int _y = rb; _y < re; _y++)\n {\n tflux_inner(ce - cb, f_v + ((0*_ny + _y)*4 + 0)*lsdf + cb, f_v + ((0*_ny + _y)*4 + 1)*lsdf + cb, f_v + ((0*_ny + _y)*4 + 2)*lsdf + cb, f_v + ((0*_ny + _y)*4 + 3)*lsdf + cb, f_v + ((1*_ny + _y)*4 + 0)*lsdf + cb, f_v + ((1*_ny + _y)*4 + 1)*lsdf + cb, f_v + ((1*_ny + _y)*4 + 2)*lsdf + cb, f_v + ((1*_ny + _y)*4 + 3)*lsdf + cb, smats_v + ((0*_ny + _y)*2 + 0)*lsdsmats + cb, smats_v + ((0*_ny + _y)*2 + 1)*lsdsmats + cb, smats_v + ((1*_ny + _y)*2 + 0)*lsdsmats + cb, smats_v + ((1*_ny + _y)*2 + 1)*lsdsmats + cb, u_v + (_y*4 + 0)*lsdu + cb, u_v + (_y*4 + 1)*lsdu + cb, u_v + (_y*4 + 2)*lsdu + cb, u_v + (_y*4 + 3)*lsdu + cb);\n }\n }\n }\n\nq\x01]q\x02(cnumpy\nint32\nq\x03h\x03cnumpy\nint64\nq\x04h\x03h\x04h\x03h\x04h\x03e\x87q\x05.', b'\x80\x03}q\x00.')

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/User/PyFR/env4/lib/python3.4/site-packages/pytools/prefork.py", line 46, in call_capture_output
popen = Popen(cmdline, cwd=cwd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
File "/usr/lib/python3.4/subprocess.py", line 859, in __init__
restore_signals, start_new_session)
File "/usr/lib/python3.4/subprocess.py", line 1395, in _execute_child
restore_signals, start_new_session, preexec_fn)
BlockingIOError: [Errno 11] Resource temporarily unavailable

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "/home/User/PyFR/env4/bin/pyfr", line 9, in <module>
load_entry_point('pyfr==1.3.0', 'console_scripts', 'pyfr')()
File "/home/User/PyFR/env4/src/pyfr/pyfr/scripts/main.py", line 109, in main
args.process(args)
File "/home/User/PyFR/env4/src/pyfr/pyfr/scripts/main.py", line 230, in process_run
args, NativeReader(args.mesh), None, Inifile.load(args.cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/scripts/main.py", line 214, in _process_common
solver = get_solver(backend, rallocs, mesh, soln, cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/solvers/__init__.py", line 14, in get_solver
return get_integrator(backend, systemcls, rallocs, mesh, initsoln, cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/integrators/__init__.py", line 26, in get_integrator
return integrator(backend, systemcls, rallocs, mesh, initsoln, cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/integrators/controllers.py", line 14, in __init__
super().__init__(*args, **kwargs)
File "/home/User/PyFR/env4/src/pyfr/pyfr/integrators/steppers.py", line 9, in __init__
super().__init__(*args, **kwargs)
File "/home/User/PyFR/env4/src/pyfr/pyfr/integrators/base.py", line 41, in __init__
self.system = systemcls(backend, rallocs, mesh, initsoln, nreg, cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/solvers/base/system.py", line 59, in __init__
self._gen_kernels(eles, int_inters, mpi_inters, bc_inters)
File "/home/User/PyFR/env4/src/pyfr/pyfr/solvers/base/system.py", line 167, in _gen_kernels
kernels[pn, kn].append(kgetter())
File "/home/User/PyFR/env4/src/pyfr/pyfr/solvers/euler/elements.py", line 70, in <lambda>
f=self._vect_upts
File "/home/User/PyFR/env4/src/pyfr/pyfr/backends/base/backend.py", line 154, in kernel
return kern(*args, **kwargs)
File "/home/User/PyFR/env4/src/pyfr/pyfr/backends/base/kernels.py", line 162, in kernel_meth
fun = self._build_kernel(name, src, list(it.chain(*argt)))
File "/home/User/PyFR/env4/src/pyfr/pyfr/util.py", line 34, in __call__
res = cache[key] = self.func(*args, **kwargs)
File "/home/User/PyFR/env4/src/pyfr/pyfr/backends/openmp/provider.py", line 13, in _build_kernel
mod = GccSourceModule(src, self.backend.cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/backends/openmp/compiler.py", line 61, in __init__
super().__init__(src, cfg)
File "/home/User/PyFR/env4/src/pyfr/pyfr/backends/openmp/compiler.py", line 30, in __init__
lname = self._build(tmpdir)
File "/home/User/PyFR/env4/src/pyfr/pyfr/backends/openmp/compiler.py", line 80, in _build
call_capture_output(cmd + self._cflags, cwd=tmpdir)
File "/home/User/PyFR/env4/lib/python3.4/site-packages/pytools/prefork.py", line 197, in call_capture_output
return forker[0].call_capture_output(cmdline, cwd, error_on_nonzero)
File "/home/User/PyFR/env4/lib/python3.4/site-packages/pytools/prefork.py", line 54, in call_capture_output
% ( " ".join(cmdline), e))
pytools.prefork.ExecError: error invoking 'gcc -shared -std=c99 -Ofast -march=native -fopenmp -fPIC -o libtmp.so tmp.c': [Errno 11] Resource temporarily unavailable

Hi Freddie,

After commenting out the prefork enabling, I made the following changes to util.py on the call function, from line 21.

def __call__(self, *args, **kwargs):
instance = args[0]
print("")
print(self)
print("--self")
print(*args)
print("--args")
print(**kwargs)
print("--kwargs")
print("")
try:
cache = instance._memoize_cache
except AttributeError:
cache = instance._memoize_cache = {}

key = (self.func, pickle.dumps(args[1:]), pickle.dumps(kwargs))

try:
res = cache[key]
print(res)
print("res cache key good")
except KeyError:
res = cache[key] = self.func(*args, **kwargs)
print("res cache key bad")
print(res)
print("new res cache key")
return res

I did this so that the shell would print out the arguments being applied, and that may indicate what is causing the error. The result came out to this: http://pastebin.com/pjJSKP8m

One line that did stand out was this one, at 1165:
0 [main] python3 4796 child_info_fork::abort: unable to remap libtmp.so to same address as parent (0x1F0000) - try running rebaseall
which may be problematic since those are tmp files, and I ran rebaseall just to be safe, but I had the same error afterwards.

Would you say that this is an OS error? I’m wondering if this can be patched with a small amount of code, but if this is beyond your realm then that’s fair enough, I’ll try looking elsewhere for solutions.

I hope to hear back from you soon.

Kind Regards,

Leo Allen

Hi Leo,

Here is your error. Cygwin does not seem to be able to correctly
emulate the position independent nature of shared libraries that is
required by the C/OpenMP backend. Instead it appears to have to rebase
.so's (much like Windows usually does with .dll's) with the shared
libraries themselves being position dependent.

The need to explicitly rebase shared libraries does not chime well with
dynamic compilation approached used by PyFR.

It is quite possibly less work to get the C/OpenMP backend working
natively with Microsoft Windows than it is to work around these defects
in Cygwin. The CUDA backend has been reported to work successfully and
so only a few minor changes related to the eager deletion of shared
libraries should be required.

Regards, Freddie.

Hi Freddie,

I have a similar problem.

I installed Anaconda distribution, therefore I am not sure about the compiler but mpi4py seems to work only with Intel MPI. At first it was installing using MS MPI with wrong directories, then I corrected them with correct MS MPI. But the result of famous mpiexec -4 hello.py was always 0. Then I tried with Intel MPI and it worked. I validated this behaviour with VS Intel Fortran and the result was the same. When I run mpiexec --version command I get Intel.

Please find attached the log files produced when I ran the Couette example.

Also could you tell me if Cblas directory is accepted or not, please? I do not know the execution order, so I cannot say if it is accepted or not.

Thank you,

I wish you a nice day,

Cheers,

Emre

29 Şubat 2016 Pazartesi 22:36:16 UTC+2 tarihinde Freddie Witherden yazdı:

MS MPI.txt (7.17 KB)

intel mpi.txt (8.4 KB)

Hi Emre,

I have a similar problem.

I installed Anaconda distribution, therefore I am not sure about the
compiler but mpi4py seems to work only with Intel MPI. At first it was
installing using MS MPI with wrong directories, then I corrected them
with correct MS MPI. But the result of famous mpiexec -4 hello.py was
always 0. Then I tried with Intel MPI and it worked. I validated this
behaviour with VS Intel Fortran and the result was the same. When I run
mpiexec --version command I get Intel.

Please find attached the log files produced when I ran the Couette example.

It is possible that the arguments for ICC on Windows are different to
those used by ICC on Linux. You should try compiling a simple piece of
C code with the arguments show in the error message to see if this works
(and if not, what needs to be changed for it to work).

Also could you tell me if Cblas directory is accepted or not, please? I
do not know the execution order, so I cannot say if it is accepted or not.

I am not sure exactly what you mean by Cblas directory? The cblas key
should have a value which is the path to a suitable cblas implementation
on your system (Intel MKL, ATLAS, OpenBLAS).

Regards, Freddie.

Regards, Freddie.

Hi Emre,

As documented in the user-guide the compiler command can be changed by,
in the configuration file, setting

[backend-openmp]
cc = compiler-command-here

Regards, Freddie.

Hi Freddie,

Do you have any suggestion how to do it?

I tried cc = icl (intel c compiler code for windows corresponding to icc of linux)
I tried cc = full path\icl.exe

I tried to launch pyfr from Intel compiler command window.

Each time I get this error:

The first part with OpenMp Kernel Provider still exists then:

File "C:\Program Files\Python35\lib\site-packages\pytools\prefork.py", line 50, in call_capture_output
% (popen.returncode, " ".join(cmdline), stderr_data))
pytools.prefork.ExecError: status 2 invoking 'icl -shared -std=c99 -Ofast -march=native -fopenmp -fPIC -o libtmp.so tmp.c': b"Intel(R) C++ Intel(R) 64 Compiler for applications running on Intel(R) 64, Version 16.0.2.180 Build 20160204\r\nCopyright (C) 1985-2016 Intel Corporation. All rights reserved.\r\nicl: command line warning #10006: ignoring unknown option '/shared'\r\nicl: command line warning #10006: ignoring unknown option '/std=c99'\r\nicl: command line warning #10006: ignoring unknown option '/Of'\r\nicl: command line warning #10121: overriding '/Os' with '/Ot'\r\nicl: command line warning #10006: ignoring unknown option '/march=native'\r\nicl: command line warning #10006: ignoring unknown option '/fopenmp'\r\nicl: command line warning #10006: ignoring unknown option '/fPIC'\r\n\r\ncompilation aborted for tmp.c (code 2)\r\n"

I tried to launch pyfr from Intel Compiler Command Window

11 Nisan 2016 Pazartesi 20:55:38 UTC+3 tarihinde Freddie Witherden yazdı: