Fluxsplit

channel/Makefile_leonardo

@@ -0,0 +1,33 @@
+# Compiler
+FC = mpif90
+LD = $(FC)
+
+# Paths (Modify if necessary)
+ROOT_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
+$(info $$ROOT_DIR is [${ROOT_DIR}])
+PARENT_DIR := $(abspath $(ROOT_DIR)/..)
+$(info $$PARENT_DIR is [${PARENT_DIR}])
+CUDECOMP_DIR = $(PARENT_DIR)/cuDecomp/build
+CUDA_DIR = /leonardo/prod/opt/compilers/cuda/12.3/none
+EXTRA_DIR = /leonardo/prod/opt/compilers/cuda/12.3/none/compat
+
+export $(CUDECOMP_DIR)/lib
+
+# Compiler and Linker Flags
+FFLAGS = -fast -acc -gpu=mem:managed -Mfree -Mpreprocess -cpp -cuda  -I$(CUDECOMP_DIR)/include/
+LIBS = -L$(CUDECOMP_DIR)/lib/ -L$(CUDA_DIR)/lib64/ -lcudecomp_fort -lcudecomp -cudalib=cufft  -lnvToolsExt
+
+# Source and Object Files
+OBJS = module.f90 main.f90 readinput.f90 readwrite.f90
+EXEC = mhit36
+
+$(EXEC):        $(OBJS)
+	$(LD) $(FFLAGS) $(OBJS) $(LIBS) -o $@
+
+clean:
+	$(RM) $(EXEC)
+
+.SUFFIXES: .o
+
+.f90.o:
+	$(FC)  -c $(FFLAGS) $<

channel/Makefile_local

@@ -0,0 +1,28 @@
+# Compiler
+FC = mpif90
+LD	= $(FC)
+RM	= /bin/rm -f
+
+# Paths (Modify if necessary)
+CUDECOMP_DIR = /home/milton/TCF36/cuDecomp/build/
+
+# Compiler and Linker Flags
+FFLAGS = -fast -acc -gpu=mem:managed -Mfree -Mpreprocess -cpp -cuda  -I$(CUDECOMP_DIR)/include
+LIBS = -L$(CUDECOMP_DIR)/lib -L/usr/local/cuda/lib64 -lcudecomp_fort -lcudecomp -cudalib=cufft 
+
+# Source and Object Files
+OBJS = module.f90 main.f90 readinput.f90 readwrite.f90
+EXEC = mhit36
+
+$(EXEC):	$(OBJS)
+	$(LD) $(FFLAGS) $(OBJS) $(LIBS) -o $@
+
+clean:
+	$(RM) $(EXEC)
+
+.SUFFIXES: .o
+
+.f90.o:
+	$(FC)  -c $(FFLAGS) $<
+
+

channel/README.md

@@ -0,0 +1,31 @@
+# Channel configuratuion
+
+Code for turbulent channel flow simulations (closed channel).
+Solution of Navier-Stokes equations + phase-field method (ACDI) + energy equation (passive scalar).
+For time integration, a fully explicit scheme is used: Euler explicit for ACDI + RK3 for Navier-Stokes and temperature
+
+## Main features at a glance:
+- No-slip boundary conditions at the two walls ($z=\pm h$)
+- Poisson solver based on FFT2 along $x$ and $y$ + TDMA along $z$.
+- Non-uniform grid along $z$, any stretching function can be used (defined in read_input.f90)
+- Domain of arbitary dimensions along all three directions
+
+**Autotuning of decomposition**  
+- Default: `pr=0` and `pc=0` → enables **automatic process decomposition**.  
+- Only input needed: **total number of MPI tasks**.  
+- No recompilation required when changing MPI processes.  
+
+**Optional features (conditional compilation)**  
+- Phase-field module: Can be enabled or disabled. Default is single-phase.  
+- Passive scaler: Can be enabled or disabled. Default is single-phase.  
+
+## Turbulent channel flow 
+- Turbulent channel flow at $Re_\tau=590$, Grid: $1536 \times 768 \times 576$
+
+![Test](../val/tcf2.png)
+
+## Nodes numbering and staggered grid
+
+X-pencil configuration:
+
+![Test](../val/grid_channel.png)

channel/binder.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+case $(( ${OMPI_COMM_WORLD_LOCAL_RANK} )) in
+0) export UCX_NET_DEVICES=mlx5_0:1 ;;
+1) export UCX_NET_DEVICES=mlx5_1:1 ;;
+2) export UCX_NET_DEVICES=mlx5_2:1 ;;
+3) export UCX_NET_DEVICES=mlx5_3:1 ;;
+esac
+
+echo Launching on $UCX_NET_DEVICES
+
+$*
+

channel/go_leo.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#SBATCH --account="IscrB_EXCEED"
+#SBATCH --job-name="cudec"
+#SBATCH --time=00:05:00
+#SBATCH --nodes=1      ##adjust
+#SBATCH --ntasks-per-node=4
+#SBATCH --gres=gpu:4
+#SBATCH --cpus-per-task=8
+#SBATCH --output=test.out
+#SBATCH -p boost_usr_prod
+#SBATCH --error=test.err
+#SBATCH --qos=boost_qos_dbg
+
+module load profile/candidate
+module load nvhpc/25.3
+module load hpcx-mpi/2.19
+CURRENT_DIR="$(pwd)"
+ROOT_DIR="$(dirname "$CURRENT_DIR")/cuDecomp/build/lib"
+echo "Using directory: $ROOT_DIR"
+export LD_LIBRARY_PATH=$ROOT_DIR:$LD_LIBRARY_PATH
+
+chmod 777 binder.sh
+mpirun -np 4 --map-by node:PE=8 --rank-by core  ./binder.sh ./mhit36

channel/input.inp

@@ -0,0 +1,21 @@
+0           ! fresh start=0 restart =1
+0           ! iteration to start from (fresh start 0 otherwise different)
+200000        ! Final time step
+1000         ! Saving frequency
+4.0         ! Streamwise domain lenght Lx
+4.0         ! Spanwise domain width Ly
+2.0         ! Wall-normal domain height Lz=2h
+0.01         ! Stretching factor quasi~0=uniform 1.5 mid-stretching and 3.0 extreme (do not use zero)
+1           ! inflow condition (0=zero, 1=perturbed 3D field, 2=read from fields)
+0           ! phase-field initial condition (0=circular drop, 1=read from fields)
+0           ! theta initial condition (0=uniform, 1=read from fields)
+0.0001      ! dt for the simulation
+0.0066666   ! mu (viscosity)
+1.000       ! density
+0.0        ! Pressure gradient along x (negative if flow along x) 
+0.0         ! Pressure gradient along y (negative if flow along y) 
+0.5         ! Initial drop radius for phase-field
+0.1d0       ! Surface tension
+1.0         ! Ratio eps/dx
+0.00666666  ! kappa (thermal diffusivity)
+50.0         ! alphag for buoyancy

channel/leo.sh

@@ -0,0 +1,8 @@
+module load profile/candidate
+module load nvhpc/25.3
+module load hpcx-mpi/2.19
+cp Makefile_leonardo Makefile
+make clean
+make
+mkdir -p output
+

channel/local.sh

@@ -0,0 +1,15 @@
+NVARCH=Linux_x86_64; export NVARCH
+NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS
+MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/25.7/compilers/man; export MANPATH
+PATH=$NVCOMPILERS/$NVARCH/25.7/compilers/bin:$PATH; export PATH
+export PATH=$NVCOMPILERS/$NVARCH/25.7/comm_libs/mpi/bin:$PATH
+export MANPATH=$MANPATH:$NVCOMPILERS/$NVARCH/25.7/comm_libs/mpi/man
+LD_LIBRARY_PATH=/home/milton/MHIT36/cuDecomp/build/lib
+#clean folder output
+rm -rf output
+mkdir output
+cp Makefile_local Makefile
+#rm *.dat
+make clean
+make
+mpirun -np 2 ./mhit36