SUBROUTINE ESETUP_TRANS(KMSMAX,KSMAX,KDGL,KDGUX,KLOEN,LDSPLIT,& & KTMAX,KRESOL,PEXWN,PEYWN,PWEIGHT,LDGRIDONLY,KNOEXTZL,KNOEXTZG, & & LDUSEFFTW,LD_ALL_FFTW) !**** *ESETUP_TRANS* - Setup transform package for specific resolution ! Purpose. ! -------- ! To setup for making spectral transforms. Each call to this routine ! creates a new resolution up to a maximum of NMAX_RESOL set up in ! SETUP_TRANS0. You need to call SETUP_TRANS0 before this routine can ! be called. !** Interface. ! ---------- ! CALL ESETUP_TRANS(...) ! Explicit arguments : KLOEN,LDSPLIT are optional arguments ! -------------------- ! KSMAX - spectral truncation required ! KDGL - number of Gaussian latitudes ! KLOEN(:) - number of points on each Gaussian latitude [2*KDGL] ! LDSPLIT - true if split latitudes in grid-point space [false] ! KTMAX - truncation order for tendencies? ! KRESOL - the resolution identifier ! KSMAX,KDGL,KTMAX and KLOEN are GLOBAL variables desribing the resolution ! in spectral and grid-point space ! LDGRIDONLY - true if only grid space is required ! LDSPLIT describe the distribution among processors of ! grid-point data and has no relevance if you are using a single processor ! LDUSEFFTW - Use FFTW for FFTs ! LD_ALL_FFTW : T to transform all fields in one call, F to transforms fields one after another ! Method. ! ------- ! Externals. ESET_RESOL - set resolution ! ---------- ESETUP_DIMS - setup distribution independent dimensions ! SUEMP_TRANS_PRELEG - first part of setup of distr. environment ! SULEG - Compute Legandre polonomial and Gaussian ! Latitudes and Weights ! ESETUP_GEOM - Compute arrays related to grid-point geometry ! SUEMP_TRANS - Second part of setup of distributed environment ! SUEFFT - setup for FFT ! Author. ! ------- ! Mats Hamrud *ECMWF* ! Modifications. ! -------------- ! Original : 00-03-03 ! 02-04-11 A. Bogatchev: Passing of TCDIS ! 02-11-14 C. Fischer: soften test on KDGL ! M.Hamrud 01-Oct-2003 CY28 Cleaning ! A.Nmiri 15-Nov-2007 Phasing with TFL 32R3 ! A.Bogatchev 16-Sep-2010 Phasing cy37 ! D. Degrauwe (Feb 2012): Alternative extension zone (E') ! R. El Khatib 02-Mar-2012 Support for mixed multi-resolutions ! R. El Khatib 09-Aug-2012 %LAM in GEOM_TYPE ! R. El Khatib 14-Jun-2013 LENABLED ! R. El Khatib 01-Sep-2015 Support for FFTW ! R. El Khatib 08-Jun-2023 LALL_FFTW for better flexibility ! ------------------------------------------------------------------ USE PARKIND1 ,ONLY : JPIM ,JPRB USE YOMHOOK ,ONLY : LHOOK, DR_HOOK, JPHOOK !ifndef INTERFACE USE TPM_GEN ,ONLY : NERR, NOUT, NPRINTLEV, MSETUP0, & & NCUR_RESOL, NDEF_RESOL, NMAX_RESOL, LENABLED USE TPM_DIM ,ONLY : R, DIM_RESOL USE TPM_DISTR ,ONLY : D, DISTR_RESOL,NPROC,NPRTRV, MYPROC USE TPM_GEOMETRY ,ONLY : G, GEOM_RESOL USE TPM_FIELDS ,ONLY : FIELDS_RESOL USE TPM_FLT ,ONLY : FLT_RESOL USE TPM_CTL ,ONLY : CTL_RESOL USE TPMALD_DIM ,ONLY : RALD, ALDDIM_RESOL USE TPMALD_DISTR ,ONLY : DALD, ALDDISTR_RESOL USE TPMALD_FIELDS ,ONLY : ALDFIELDS_RESOL, FALD USE TPMALD_GEO ,ONLY : GALD, ALDGEO_RESOL USE ESET_RESOL_MOD ,ONLY : ESET_RESOL USE ESETUP_DIMS_MOD ,ONLY : ESETUP_DIMS USE SUEMP_TRANS_MOD ,ONLY : SUEMP_TRANS USE SUEMP_TRANS_PRELEG_MOD ,ONLY : SUEMP_TRANS_PRELEG USE ESETUP_GEOM_MOD ,ONLY : ESETUP_GEOM USE ABORT_TRANS_MOD ,ONLY : ABORT_TRANS #ifdef ACCGPU USE OPENACC #endif #ifdef OMPGPU USE OMP_LIB #endif !endif INTERFACE IMPLICIT NONE ! Dummy arguments INTEGER(KIND=JPIM),INTENT(IN) :: KMSMAX INTEGER(KIND=JPIM),INTENT(IN) :: KSMAX INTEGER(KIND=JPIM),INTENT(IN) :: KDGL INTEGER(KIND=JPIM),INTENT(IN) :: KDGUX INTEGER(KIND=JPIM),INTENT(IN) :: KLOEN(:) LOGICAL ,OPTIONAL,INTENT(IN) :: LDSPLIT LOGICAL ,OPTIONAL,INTENT(IN) :: LDGRIDONLY INTEGER(KIND=JPIM),OPTIONAL,INTENT(IN) :: KTMAX INTEGER(KIND=JPIM),OPTIONAL,INTENT(INOUT) :: KRESOL REAL(KIND=JPRB) ,OPTIONAL,INTENT(IN) :: PEXWN REAL(KIND=JPRB) ,OPTIONAL,INTENT(IN) :: PEYWN REAL(KIND=JPRB) ,OPTIONAL,INTENT(IN) :: PWEIGHT(:) INTEGER(KIND=JPIM),OPTIONAL,INTENT(IN) :: KNOEXTZL INTEGER(KIND=JPIM),OPTIONAL,INTENT(IN) :: KNOEXTZG LOGICAL ,OPTIONAL,INTENT(IN) :: LDUSEFFTW LOGICAL ,OPTIONAL,INTENT(IN) :: LD_ALL_FFTW !ifndef INTERFACE ! Local variables LOGICAL :: LLP1,LLP2 #ifdef ACCGPU INTEGER(ACC_DEVICE_KIND) :: IDEVTYPE #endif INTEGER :: INUMDEVS, IUNIT, ISTAT, IDEV, MYGPU INTEGER :: I, J REAL(KIND=JPHOOK) :: ZHOOK_HANDLE ! ------------------------------------------------------------------ IF (LHOOK) CALL DR_HOOK('ESETUP_TRANS',0,ZHOOK_HANDLE) IF(MSETUP0 == 0) THEN CALL ABORT_TRANS('ESETUP_TRANS: SETUP_TRANS0 HAS TO BE CALLED BEFORE ESETUP_TRANS') ENDIF LLP1 = NPRINTLEV>0 LLP2 = NPRINTLEV>1 IF(LLP1) WRITE(NOUT,*) '=== ENTER ROUTINE ESETUP_TRANS ===' ! Allocate resolution dependent structures common to global and LAM IF(.NOT. ALLOCATED(DIM_RESOL)) THEN NDEF_RESOL = 1 ALLOCATE(DIM_RESOL(NMAX_RESOL)) ALLOCATE(FIELDS_RESOL(NMAX_RESOL)) ALLOCATE(GEOM_RESOL(NMAX_RESOL)) ALLOCATE(DISTR_RESOL(NMAX_RESOL)) ALLOCATE(FLT_RESOL(NMAX_RESOL)) ALLOCATE(CTL_RESOL(NMAX_RESOL)) GEOM_RESOL(:)%LAM=.FALSE. ALLOCATE(LENABLED(NMAX_RESOL)) LENABLED(:)=.FALSE. ELSE NDEF_RESOL = NDEF_RESOL+1 IF(NDEF_RESOL > NMAX_RESOL) THEN CALL ABORT_TRANS('ESETUP_TRANS:NDEF_RESOL > NMAX_RESOL') ENDIF ENDIF ! Allocate LAM-specific resolution dependent structures IF(.NOT. ALLOCATED(ALDDIM_RESOL)) THEN ALLOCATE(ALDDIM_RESOL(NMAX_RESOL)) ALLOCATE(ALDFIELDS_RESOL(NMAX_RESOL)) ALLOCATE(ALDGEO_RESOL(NMAX_RESOL)) ALLOCATE(ALDDISTR_RESOL(NMAX_RESOL)) ENDIF IF (PRESENT(KRESOL)) THEN KRESOL=NDEF_RESOL ENDIF ! Point at structures due to be initialized CALL ESET_RESOL(NDEF_RESOL) IF(LLP1) WRITE(NOUT,*) '=== DEFINING RESOLUTION ',NCUR_RESOL ! Defaults for optional arguments G%LREDUCED_GRID = .FALSE. D%LGRIDONLY = .FALSE. D%LSPLIT = .FALSE. ! NON-OPTIONAL ARGUMENTS R%NSMAX = KSMAX RALD%NMSMAX=KMSMAX RALD%NDGUX=KDGUX R%NDGL = KDGL RALD%NDGLSUR=KDGL+2 R%NDLON =KLOEN(1) RALD%NLOENS_LON=(/ R%NDLON /) RALD%NOFFSETS_LON=(/ 0 , R%NDLON+2 /) RALD%NLOENS_LAT=(/ R%NDGL /) RALD%NOFFSETS_LAT=(/ 0 , R%NDGL+2 /) ! IMPLICIT argument : G%LAM = .TRUE. IF (KDGL <= 0) THEN CALL ABORT_TRANS ('ESETUP_TRANS: KDGL IS NOT A POSITIVE NUMBER') ENDIF ! Optional arguments ALLOCATE(G%NLOEN(R%NDGL)) IF(LLP2)WRITE(NOUT,9) 'NLOEN ',SIZE(G%NLOEN ),SHAPE(G%NLOEN ) IF (G%LREDUCED_GRID) THEN G%NLOEN(:) = KLOEN(1:R%NDGL) ELSE G%NLOEN(:) = R%NDLON ENDIF IF(PRESENT(LDSPLIT)) THEN D%LSPLIT = LDSPLIT ENDIF IF(PRESENT(KTMAX)) THEN R%NTMAX = KTMAX ELSE R%NTMAX = R%NSMAX ENDIF IF(R%NTMAX /= R%NSMAX) THEN !This SHOULD work but I don't know how to test it /MH WRITE(NERR,*) 'R%NTMAX /= R%NSMAX',R%NTMAX,R%NSMAX CALL ABORT_TRANS('ESETUP_TRANS:R%NTMAX /= R%NSMAX HAS NOT BEEN VALIDATED') ENDIF IF(PRESENT(PWEIGHT)) THEN D%LWEIGHTED_DISTR = .TRUE. IF( D%LWEIGHTED_DISTR .AND. .NOT.D%LSPLIT )THEN CALL ABORT_TRANS('SETUP_TRANS: LWEIGHTED_DISTR=T AND LSPLIT=F NOT SUPPORTED') ENDIF IF(SIZE(PWEIGHT) /= SUM(G%NLOEN(:)) )THEN CALL ABORT_TRANS('SETUP_TRANS:SIZE(PWEIGHT) /= SUM(G%NLOEN(:))') ENDIF ALLOCATE(D%RWEIGHT(SIZE(PWEIGHT))) D%RWEIGHT(:)=PWEIGHT(:) ELSE D%LWEIGHTED_DISTR = .FALSE. ENDIF IF(PRESENT(LDGRIDONLY)) THEN D%LGRIDONLY=LDGRIDONLY ENDIF IF (PRESENT(KNOEXTZL)) THEN R%NNOEXTZL=KNOEXTZL ELSE R%NNOEXTZL=0 ENDIF IF (PRESENT(KNOEXTZG)) THEN R%NNOEXTZG=KNOEXTZG ELSE R%NNOEXTZG=0 ENDIF ! Setup resolution dependent structures ! ------------------------------------- ! Setup distribution independent dimensions CALL ESETUP_DIMS IF (PRESENT(PEXWN)) GALD%EXWN=PEXWN IF (PRESENT(PEYWN)) GALD%EYWN=PEYWN ! GPU stuff: from setup_trans.F90 #ifdef ACCGPU IDEVTYPE=ACC_GET_DEVICE_TYPE() INUMDEVS = ACC_GET_NUM_DEVICES(IDEVTYPE) MYGPU = MOD(MYPROC-1,INUMDEVS) CALL ACC_SET_DEVICE_NUM(MYGPU, IDEVTYPE) MYGPU = ACC_GET_DEVICE_NUM(IDEVTYPE) !ISTAT = CUDA_GETDEVICE(IDEV) #endif #ifdef ACCGPU !$ACC ENTER DATA COPYIN(D,R,G) !$ACC ENTER DATA & !$ACC& COPYIN(D%NUMP,D%MYMS,D%NPNTGTB0,D%NPNTGTB1,D%NSTAGT0B,D%NSTAGT1B,D%NSTAGTF,D%NPROCM,D%NPTRLS,D%MSTABF) & !$ACC& COPYIN(R%NDGNH,R%NSMAX) & !$ACC& COPYIN(G%NDGLU,G%NMEN,G%NLOEN) #endif #ifdef OMPGPU !$OMP TARGET ENTER DATA MAP(TO:F,D,D%NUMP,D%MYMS,R,R%NDGNH,R%NSMAX,G,G%NDGLU) !$OMP TARGET ENTER DATA MAP(TO:D%NPNTGTB0,D%NPNTGTB1,D%NSTAGT0B,D%NSTAGT1B,D%NSTAGTF,G%NMEN,D%NPROCM,D%NPTRLS,G,G%NLOEN,D%MSTABF) #endif ! First part of setup of distributed environment CALL SUEMP_TRANS_PRELEG CALL GSTATS(1802,0) ! Compute arrays related to grid-point geometry CALL ESETUP_GEOM ! Second part of setup of distributed environment CALL SUEMP_TRANS ! Initialize Fast Fourier Transform package CALL GSTATS(1802,1) ! Signal the current resolution is active LENABLED(NDEF_RESOL)=.TRUE. #ifdef ACCGPU WRITE(NOUT,*) 'Using OpenACC' #endif #ifdef OMPGPU WRITE(NOUT,*) 'Using OpenMP offloading' #endif #ifdef ACCGPU !$ACC ENTER DATA COPYIN(R,D,G,DALD,FALD,RALD) !$ACC ENTER DATA COPYIN(R%NSMAX,R%NTMAX,R%NDGL,R%NDGNH,D%NSTAGT0B,D%NSTAGT1B,& !$ACC& D%NPNTGTB1,D%NPROCL,D%NUMP,D%NDGL_FS,D%MYMS,D%NASM0,D%NSTAGTF,D%MSTABF,& !$ACC& D%NPNTGTB0,D%NPROCM,D%NPTRLS,G%NDGLU,G%NMEN,G%NLOEN,& !$ACC& DALD%NESM0,DALD%NCPL2M, DALD%NPME, FALD%RLEPINM, & !$ACC& RALD%NLOENS_LON,RALD%NOFFSETS_LON,RALD%NLOENS_LAT,RALD%NOFFSETS_LAT) #endif #ifdef OMPGPU !$OMP TARGET ENTER DATA MAP(TO:R%NSMAX,R%NTMAX,R%NDGL,R%NDGNH,D%NSTAGT0B,D%NSTAGT1B) !$OMP TARGET ENTER DATA MAP(TO:D%NPNTGTB1,D%NPROCL,D%NUMP,D%NDGL_FS,D%MYMS,D%NASM0,D%NSTAGTF,D%MSTABF) !$OMP TARGET ENTER DATA MAP(TO:D%NPNTGTB0,D%NPROCM,D%NPTRLS,G%NDGLU,G%NMEN,G%NLOEN) !$OMP TARGET ENTER DATA MAP(TO:DALD%NESM0,DALD%NCPL2M, DALD%NPME, FALD%RLEPINM) !$OMP TARGET ENTER DATA MAP(TO:RALD%NLOENS_LON,RALD%NOFFSETS_LON,RALD%NLOENS_LAT,RALD%NOFFSETS_LAT #endif WRITE(NOUT,*) '===GPU arrays successfully allocated' #ifdef ACCGPU !$ACC wait #endif #ifdef OMPGPU !$OMP BARRIER #endif IF (LHOOK) CALL DR_HOOK('ESETUP_TRANS',1,ZHOOK_HANDLE) ! ------------------------------------------------------------------ 9 FORMAT(1X,'ARRAY ',A10,' ALLOCATED ',8I8) !endif INTERFACE END SUBROUTINE ESETUP_TRANS