From 684f58302f2fe9cde25d82f8339a2d7a47baaaed Mon Sep 17 00:00:00 2001 From: Tao Sun <41139193+mos3r3n@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:36:12 -0600 Subject: [PATCH 01/30] Bugfix for duplicated allocation and file closing for radar_rhv_opt = 2 (#2078) TYPE: bug fix KEYWORDS: radar DA DESCRIPTION OF CHANGES: Problem: Duplicated allocation of avg_qws when choosing radar_rhv_opt = 2. da_free_unit instead of da_get_unit for closing a file when radar_rhv_opt is equal to 2. ISSUE: #2077 LIST OF MODIFIED FILES: var/da/da_radar/da_get_innov_vector_radar.inc --- var/da/da_radar/da_get_innov_vector_radar.inc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/var/da/da_radar/da_get_innov_vector_radar.inc b/var/da/da_radar/da_get_innov_vector_radar.inc index bddcda85c3..4b7b1b9fb2 100644 --- a/var/da/da_radar/da_get_innov_vector_radar.inc +++ b/var/da/da_radar/da_get_innov_vector_radar.inc @@ -275,7 +275,6 @@ END IF allocate (avg_qrn(tot_h_index,tot_z_index)) allocate (avg_qds(tot_h_index,tot_z_index)) allocate (avg_qws(tot_h_index,tot_z_index)) - allocate (avg_qws(tot_h_index,tot_z_index)) allocate (avg_qgr(tot_h_index,tot_z_index)) allocate (ave_rho(tot_h_index,tot_z_index)) @@ -379,7 +378,7 @@ END IF end do end do !bottom-top if (rootproc) close(hydro_weight_unit) - if (rootproc) call da_get_unit(hydro_weight_unit) + if (rootproc) call da_free_unit(hydro_weight_unit) end if !! use_radar_rhv .and. radar_rhv_opt == 2 do n=iv%info(radar)%n1,iv%info(radar)%n2 From 727d2537a43ab5d4d35eed3957d6fc771cc59b93 Mon Sep 17 00:00:00 2001 From: weiwangncar Date: Tue, 13 Aug 2024 02:04:31 -0600 Subject: [PATCH 02/30] a few changes in documenting namelists (#2074) TYPE: text only KEYWORDS: namelists, corrections SOURCE: JiriRichter and internal DESCRIPTION OF CHANGES: The description of a few namelists are updated. io_style_emiss is removed from this file since it is for WRF-Chem, and not specifiable under &time_control. Namelist ntiedtke_dx_opt was also removed, since it is not implemented. ISSUE: For use when this PR closes an issue. Fixes #2071, #2072, #2073 LIST OF MODIFIED FILES: run/README.namelist RELEASE NOTE: The documentation of a few namelists is updated in this PR. --- run/README.namelist | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/run/README.namelist b/run/README.namelist index be2a30b145..4f1d31f71b 100644 --- a/run/README.namelist +++ b/run/README.namelist @@ -60,11 +60,6 @@ Description of namelist variables = 10, ! GRIB2 format = 11, ! pnetCDF format ncd_nofill = .true., ! only a single write, not the write/read/write sequence - frames_per_emissfile = 12, ! number of times in each chemistry emission file. - io_style_emiss = 1, ! style to use for the chemistry emission files. - ! 0 = Do not read emissions from files. - ! 1 = Cycle between two 12 hour files (set frames_per_emissfile=12) - ! 2 = Dated files with length set by frames_per_emissfile debug_level = 0, ! 50,100,200,300 values give increasing prints diag_print = 0, ! print out time series of model diagnostics 0 = no print @@ -89,20 +84,20 @@ To choose between SI and WPS input to real for EM core: auxinput1_inname = "met_em.d." ! Input to real from WPS (default since 3.0) = "wrf_real_input_em.d." ! Input to real from SI -Other output options: +Other output options: Note all auxhist[1-24], auxinput[2-24] interval variables are domain dependent - auxhist2_outname = "rainfall" ! file name for extra output! if not specified, - auxhist2_d_ will be used + auxhist9_outname = "rainfall" ! file name for extra output! if not specified, + auxhist9_d_ will be used also note that to write variables in output other than the history file requires Registry.EM file change - auxhist2_interval (max_dom) = 10, ! interval in minutes - io_form_auxhist2 = 2, ! output in netCDF - frames_per_auxhist2 = 1000, ! number of output times in this file + auxhist9_interval (max_dom) = 10, ! interval in minutes + io_form_auxhist9 = 2, ! output in netCDF + frames_per_auxhist9 = 1000, ! number of output times in this file For SST updating (used only with sst_update=1): auxinput4_inname = "wrflowinp_d" - auxinput4_interval = 360 ! minutes generally matches time given by interval_seconds + auxinput4_interval (max_dom) = 360 ! minutes generally matches time given by interval_seconds io_form_auxinput4 = 2 ! IO format nwp_diagnostics = 0 ! set to = 1 to add 7 history-interval max diagnostic fields @@ -112,7 +107,7 @@ For additional regional climate surface fields output_diagnostics = 0 ! set to = 1 to add 36 surface diagnostic arrays (max/min/mean/std) auxhist3_outname = 'wrfxtrm_d_' ! file name for added diagnostics io_form_auxhist3 = 2 ! netcdf - auxhist3_interval = 1440 ! minutes between outputs (1440 gives daily max/min) + auxhist3_interval (max_dom) = 1440 ! minutes between outputs (1440 gives daily max/min) frames_per_auxhist3 = 1 ! output times per file Note: do restart only at multiple of auxhist3_intervals @@ -138,13 +133,13 @@ Additional settings when running WRFVAR: inputout_interval (max_dom) = 180, ! interval in minutes when writing input-formatted data input_outname = 'wrfinput_d_' ! you may change the output file name inputout_begin_y (max_dom) = 0 - inputout_begin_mo = 0 + inputout_begin_m = 0 inputout_begin_d (max_dom) = 0 inputout_begin_h (max_dom) = 3 inputout_begin_m (max_dom) = 0 inputout_begin_s (max_dom) = 0 inputout_end_y (max_dom) = 0 - inputout_end_mo = 0 + inputout_end_m = 0 inputout_end_d (max_dom) = 0 inputout_end_h (max_dom) = 12 inputout_end_m (max_dom) = 0 @@ -495,7 +490,7 @@ Namelist variables for controlling the adaptive time step option: For NSSL 1-moment schemes, intercept and particle densities can be set for snow, graupel, hail, and rain. For the 1- and 2-moment schemes, the shape parameters for graupel and hail can be set. - PLEASE SEE README.NSSLmp for options affecting the NSSL scheme + PLEASE SEE doc/README.NSSLmp for options affecting the NSSL scheme = 17, 19, 21, 22: Legacy NSSL-MP options: see README.NSSLmp for equivalent settings with 18 = 24, WSM 7-class scheme (separate hail and graupel categories) = 26, WDM 7-class scheme (separate hail and graupel categories) @@ -874,9 +869,6 @@ Namelist variables for controlling the adaptive time step option: = 3, for small grid distances (DX < 5 km) nsas_dx_factor = 0, ! default option = 1, NSAS grid-distance dependent option - ntiedtke_dx_opt = 0, default option - = 1, new Tiedtke grid-distance dependent option (scale-aware) - when grid size falls below 15 km For KF-CuP scheme: recommended to use with cu_rad_feedback shallowcu_forced_ra(max_dom) radiative impact of shallow Cu by a prescribed maximum cloud fraction = .false., option off, default From ae6c32144de9f32c1a92f3db5db84a487bfd90e3 Mon Sep 17 00:00:00 2001 From: joshi994 <160153650+joshi994@users.noreply.github.com> Date: Mon, 19 Aug 2024 22:21:47 -0700 Subject: [PATCH 03/30] Bug fix for LECH's stability functions in the SL Urban Canopy Model. (#2032) TYPE: Bug fix KEYWORDS: LECH stability functions, Latent heat flux, Sensible heat flux, SL Urban Canpy Model SOURCE: Parag Joshi (Brookhaven National Lab), Katia Lamer (Brookhaven National Lab) DESCRIPTION OF CHANGES: Problem: The stability functions originally proposed by Lech Lobocki which are used in calculating the exchange coefficients between surface layer and roof, building wall, Green roof, and ground are implemented incorrectly. The error surfaced while comparing the formulation suggested in Lobocki 1992 with the equations in commands/lines 3056 and 3062 (WRF-v4.5.2). Solution: The code has been corrected by referring to the equations 48 and 49 of the article "A procedure for the derivation of surface layer bulk relationships from simplified second-order closure models" by Lobocki 1992 (Journal of Applied Meteorology). LIST OF MODIFIED FILES: M phys/module_sf_urban.F TESTS CONDUCTED: 1. The significance of the corrections can be highlighted by comparing a test run with the corrected module and current version of the WRF model. 2. The fix will correct the evaluation of the exchange coefficients between the ground, roof, green roof etc and air. 3. The Jenkins tests are all passing. RELEASE NOTE: Lech's stability functions are corrected following the original formulation in the single layer urban canopy model. --- phys/module_sf_urban.F | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/phys/module_sf_urban.F b/phys/module_sf_urban.F index 193e1cfda1..2a5805b917 100644 --- a/phys/module_sf_urban.F +++ b/phys/module_sf_urban.F @@ -66,7 +66,6 @@ MODULE module_sf_urban REAL, ALLOCATABLE, DIMENSION(:) :: TRLEND_TBL, TBLEND_TBL, TGLEND_TBL REAL, ALLOCATABLE, DIMENSION(:) :: AKANDA_URBAN_TBL !for BEP - ! MAXDIRS :: The maximum number of street directions we're allowed to define INTEGER, PARAMETER :: MAXDIRS = 3 ! MAXHGTS :: The maximum number of building height bins we're allowed to define @@ -3160,13 +3159,13 @@ SUBROUTINE SFCDIF_URB (ZLM,Z0,THZ0,THLM,SFCSPD,AKANDA,AKMS,AKHS,RLMO,CD,ZT_OUT,V ! LECH'S SURFACE FUNCTIONS ! ---------------------------------------------------------------------- PSLMU (ZZ)= -0.96* log (1.0-4.5* ZZ) - PSLMS (ZZ)= ZZ * RRIC -2.076* (1. -1./ (ZZ +1.)) + PSLMS (ZZ)= (ZZ/RFC) -2.076* (1. -1./ (ZZ +1.)) PSLHU (ZZ)= -0.96* log (1.0-4.5* ZZ) ! ---------------------------------------------------------------------- ! PAULSON'S SURFACE FUNCTIONS ! ---------------------------------------------------------------------- - PSLHS (ZZ)= ZZ * RFAC -2.076* (1. -1./ (ZZ +1.)) + PSLHS (ZZ)= ZZ * RFAC -2.076* (1. - exp(-1.2 * ZZ)) PSPMU (XX)= -2.* log ( (XX +1.)*0.5) - log ( (XX * XX +1.)*0.5) & & +2.* ATAN (XX) & &- PIHF From 6273c88c8b992f94cd97a7a0d50165be38a1edce Mon Sep 17 00:00:00 2001 From: joshi994 <160153650+joshi994@users.noreply.github.com> Date: Mon, 19 Aug 2024 22:44:44 -0700 Subject: [PATCH 04/30] Bug fix for parameter used in calculating the heat transfer coefficient. (#2049) TYPE: Bug fix KEYWORDS: PV panels, Correlations, Upward heat flow, BEM (Building Energy Model). SOURCE: Parag Joshi (Brookhaven National Laboratory) DESCRIPTION OF CHANGES: Problem: A typographical mistake was identified within the correlations that evaluate heat transfer coefficient for the upward heat flow over PV panels in the Building Energy Model (BEM). Solution: The parameters have been modified following the correlation expressions provided in the literature. Please refer to the equation J.2.2a on page number 79 of the document below: https://nvlpubs.nist.gov/nistpubs/Legacy/IR/nbsir83-2655.pdf LIST OF MODIFIED FILES: M phys/module_sf_bem.F TESTS CONDUCTED: 1. No test has been conducted as the parameters seems to have very small or insignificant impact on the results. However, the tests can be performed with and without correcting the parameters to quantify the role. 2. The Jenkins tests are all passing. RELEASE NOTE: Bug fixes for parameters associated with the heat transfer coefficient for the upward heat flow over PV panels. --- phys/module_sf_bem.F | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phys/module_sf_bem.F b/phys/module_sf_bem.F index e693e7c6d2..d9ce496f31 100644 --- a/phys/module_sf_bem.F +++ b/phys/module_sf_bem.F @@ -889,7 +889,7 @@ subroutine hsfluxpv(nz,n,bl,bw,albr,rs,swddif,emr,rl,tr,tair,sigma,hspv,eppv,pv_ uroof=(uout(n+1)**2+vout(n+1)**2)**0.5 deltat=tpv-tair(n+1) hf=2.5*(40./100.*uroof)**(0.5) - hc=9.842*abs(deltat)**(1./3.)/(7.283-abs(cos(tiltangle))) + hc=9.482*abs(deltat)**(1./3.)/(7.238-abs(cos(tiltangle))) hup=sqrt(hc**2.+(hf)**2.) hc=1.810*abs(deltat)**(1./3.)/(1.382+abs(cos(tiltangle))) hdown=sqrt(hc**2.+(hf)**2.) From 6198090842e05f4c3884fd7f1162bb6ebe97175a Mon Sep 17 00:00:00 2001 From: joshi994 <160153650+joshi994@users.noreply.github.com> Date: Tue, 20 Aug 2024 19:48:18 -0700 Subject: [PATCH 05/30] Bug fixes for stability function and energy balance in single canopy urban model (#2038) TYPE: Bug fix KEYWORDS: Stability function, Urban canopy model, surface energy balance, Green roof temperature. SOURCE: Parag Joshi (Brookhaven National Laboratory), Katia Lamer (Brookhaven National Laboratory) DESCRIPTION OF CHANGES: Problem: The calculations for PSIM in the mos subroutine of module-sf_urban.F has been corrected. The earlier equations has a typo where X (= (1-16\zeta)^1/4) is mentioned instead of X0. In addition, following the formulation of surface energy balance of OSU1DPBL scheme by Ek and Mahrt 1991, it appears that when the following definition is used $\sigma * \theta_s^4 = \sigma T_0^4 (1 + 4((\theta_s - T_0)/T_0)))$ is used, the temperature in the calculations of net flux at the surface should use T_0 (temperature at first atmospheric level). However, in the module TGRP (temperature of the green roof surface at previous time step is used)is used instead of T_0 (please refer to the attached screenshots of the formulation). Solution: Following the formulation, the temperature variable has been updated according. Moreover, the mos subroutine is also corrected by replacing X by X0 where X0 is =X*z0/(z+z0) (as per the WRF/phys/module_sf_urban.F). LIST OF MODIFIED FILES: M phys/module_sf_urban.F TESTS CONDUCTED: 1. No tests are conducted. A test case could be run to compare the impact of changes in the TGR and stability parameter value as well as CD, US, and ALPHA. 2. The Jenkins tests are all passing. RELEASE NOTE: Fixes for the calculations for PSIM in the mos subroutine and the temperature used in surface energy balance for the green roof. The later follows the OSU1DPBL document located at: [https://ftp.emc.ncep.noaa.gov/mmb/gcp/ldas/nceplsm/OSU1DPBL/OSU1DPBL-userguide.pdf] --- phys/module_sf_urban.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/phys/module_sf_urban.F b/phys/module_sf_urban.F index 2a5805b917..42806772c3 100644 --- a/phys/module_sf_urban.F +++ b/phys/module_sf_urban.F @@ -1184,7 +1184,7 @@ SUBROUTINE urban(LSOLAR, & ! L CALL TDFCND (DF1,SMR(KZ), QUARTZ, SMCMAX ) DF1 = DF1 * EXP(-2.0 * SHDFAC) - RGR = EPSV*(RX-SIG*(TGRP**4.)/60.) + RGR = EPSV*(RX-SIG*(TA**4.)/60.) RGRR= (SGR+RGR) * 697.7 * 60. RCH = RHOO*CPP*CHGR RR1 = EPSV*(TA**4) * 6.48E-8 / (PS* CHGR) + 1.0 @@ -1727,7 +1727,7 @@ SUBROUTINE mos(XXX,ALPHA,CD,B1,RIB,Z,Z0,UA,TA,TSF,RHO) PSIM=ALOG((Z+Z0)/Z0) & -ALOG((X+1.)**2.*(X**2.+1.)) & +2.*ATAN(X) & - +ALOG((X+1.)**2.*(X0**2.+1.)) & + +ALOG((X0+1.)**2.*(X0**2.+1.)) & -2.*ATAN(X0) FAIH=1./SQRT(1.-16.*XXX) PSIH=ALOG((Z+Z0)/Z0)+0.4*B1 & From 2e885eb1848cee4aedacf5dd69e4aaa578d06b17 Mon Sep 17 00:00:00 2001 From: Tanya Spero Date: Wed, 21 Aug 2024 23:26:04 -0400 Subject: [PATCH 06/30] Updated global attributes for auxiliary output as well as runs with WRF-CMAQ (#2044) TYPE: bug fix KEYWORDS: metadata, global attributes, auxiliary output, auxiliary history, WRF-CMAQ SOURCE: Tanya Spero and Jeff Willison (U.S. EPA) DESCRIPTION OF CHANGES: Problem: The suite of global attributes is incomplete in auxiliary history files. Notably, the precipitation tipping bucket is among the fields not included. Separately, there is no metadata in the header to indicate when WRF-CMAQ is used. Solution: Changes are introduced in share/output_wrf.F to expand the use cases to write the full suite of global attributes to the metadata of auxiliary output files. In addition, two global attributes are added to the output only when the WRF-CMAQ model is invoked (WRF_CMAQ_OPTION and DIRECT_SW_FEEDBACK). ISSUE: Fixes #1476 LIST OF MODIFIED FILES: share/output_wrf.F TESTS CONDUCTED: 1. Tests were conducted using the WRF model both with and without compiling with CMAQ. The base code was pulled from the branch "release-v4.6.0" on 8 May 2024. In both sets of tests on the EPA supercomputing system, an auxiliary output file was created in auxhist7. The global attributes were properly expanded to include the full suite of attributes in the auxiliary output file. The two new WRF-CMAQ coupled model attributes appeared when the WRF-CMAQ model was invoked. 2. The Jenkins tests are all passing. RELEASE NOTE: Expanded global attributes to the full suite for auxiliary history files. Also added two global attributes when the WRF-CMAQ model is invoked (WRF_CMAQ_OPTION and DIRECT_SW_FEEDBACK). --- share/output_wrf.F | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/share/output_wrf.F b/share/output_wrf.F index 3cec620bc7..69ebcf31fc 100644 --- a/share/output_wrf.F +++ b/share/output_wrf.F @@ -40,6 +40,10 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) mp_physics, ra_lw_physics, ra_sw_physics, sf_sfclay_physics, & sf_surface_physics, bl_pbl_physics, cu_physics, hypsometric_opt, sf_lake_physics, & use_theta_m, use_maxw_level, use_trop_level,hybrid_opt, gwd_opt +#if ( WRF_CMAQ == 1 ) + INTEGER wrf_cmaq_option + LOGICAL direct_sw_feedback +#endif INTEGER swint_opt, aer_type,aer_aod550_opt,aer_angexp_opt,aer_ssa_opt,aer_asy_opt, aer_opt REAL aer_aod550_val,aer_angexp_val,aer_ssa_val,aer_asy_val,etac REAL khdif, kvdif, swrad_scat, dampcoef,radt,bldt,cudt @@ -88,6 +92,7 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) CHARACTER*80 char_junk CHARACTER(LEN=256) :: MMINLU INTEGER ibuf(1) + LOGICAL lbuf(1) REAL rbuf(1) TYPE(WRFU_TimeInterval) :: bdy_increment TYPE(WRFU_Time) :: next_time, currentTime, startTime @@ -138,6 +143,10 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) call nl_get_khdif ( grid%id, khdif ) call nl_get_kvdif ( grid%id, kvdif ) call nl_get_mp_physics ( grid%id, mp_physics ) +#if ( WRF_CMAQ == 1 ) + call nl_get_wrf_cmaq_option ( grid%id, wrf_cmaq_option ) + call nl_get_direct_sw_feedback ( grid%id, direct_sw_feedback ) +#endif call nl_get_ra_lw_physics ( grid%id, ra_lw_physics ) call nl_get_ra_sw_physics ( grid%id, ra_sw_physics ) call nl_get_sf_sfclay_physics ( grid%id, sf_sfclay_physics ) @@ -347,7 +356,8 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) CALL wrf_put_dom_ti_char ( fid , 'START_DATE', TRIM(start_date) , ierr ) IF ( switch .EQ. input_only) THEN CALL wrf_put_dom_ti_char ( fid , 'SIMULATION_START_DATE', TRIM(start_date) , ierr ) - ELSE IF ( ( switch .EQ. restart_only ) .OR. ( switch .EQ. history_only ) ) THEN + ELSE IF ( ( switch .EQ. restart_only ) .OR. ( switch .EQ. history_only ) .OR. & + ( ( switch .GE. auxhist1_only ) .AND. ( switch .LE. auxhist24_only ) ) ) THEN CALL nl_get_simulation_start_year ( 1, simulation_start_year ) CALL nl_get_simulation_start_month ( 1, simulation_start_month ) CALL nl_get_simulation_start_day ( 1, simulation_start_day ) @@ -552,7 +562,8 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) END IF END IF - IF (switch .EQ. history_only) THEN + IF ( ( switch .EQ. history_only ) .OR. & + ( ( switch .GE. auxhist1_only ) .AND. ( switch .LE. auxhist24_only ) ) ) THEN CALL wrf_put_dom_ti_integer( fid, 'SKEBS_ON' , config_flags%skebs_on , 1, ierr ) IF ( config_flags%skebs_on .NE. 0 ) THEN CALL wrf_put_dom_ti_real ( fid, 'TOT_BACKSCAT_PSI' , config_flags%tot_backscat_psi , 1, ierr ) @@ -623,6 +634,13 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) CALL wrf_put_dom_ti_real ( fid , 'KHDIF' , rbuf , 1 , ierr ) rbuf(1) = kvdif CALL wrf_put_dom_ti_real ( fid , 'KVDIF' , rbuf , 1 , ierr ) +#if ( WRF_CMAQ == 1 ) + ibuf(1) = wrf_cmaq_option + CALL wrf_put_dom_ti_integer ( fid , 'WRF_CMAQ_OPTION' , ibuf , 1 , ierr ) + lbuf(1) = direct_sw_feedback +! Note the code will turn logical T/F to integer 1/0 in the output + CALL wrf_put_dom_ti_logical ( fid , 'DIRECT_SW_FEEDBACK' , lbuf , 1 , ierr ) +#endif ibuf(1) = mp_physics CALL wrf_put_dom_ti_integer ( fid , 'MP_PHYSICS' , ibuf , 1 , ierr ) ibuf(1) = ra_lw_physics @@ -672,7 +690,8 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) CALL wrf_put_dom_ti_integer ( fid, 'DISTRIBUTED_AHE_OPT', config_flags%distributed_ahe_opt, 1, ierr) #endif - IF ( switch .EQ. history_only ) THEN + IF ( ( switch .EQ. history_only ) .OR. & + ( ( switch .GE. auxhist1_only ) .AND. ( switch .LE. auxhist24_only ) ) ) THEN CALL wrf_put_dom_ti_integer ( fid, 'SHCU_PHYSICS', config_flags%shcu_physics , 1 , ierr ) CALL wrf_put_dom_ti_integer ( fid, 'MFSHCONV', config_flags%mfshconv , 1 , ierr ) CALL wrf_put_dom_ti_integer ( fid, 'FEEDBACK', feedback , 1 , ierr ) @@ -800,7 +819,7 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) CALL wrf_put_dom_ti_integer ( fid, 'NTASKS_Y', ntasks_y , 1 , ierr ) CALL wrf_put_dom_ti_integer ( fid, 'NTASKS_TOTAL', ntasks_x*ntasks_y , 1 , ierr ) - ENDIF ! history_only + ENDIF ! history_only and aux history #if (WRF_CHEM == 1 && WRF_KPP == 1) IF ( switch == auxhist9_only .and. config_flags%irr_opt /= 0 ) THEN @@ -812,6 +831,7 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) #if (EM_CORE == 1) && ( DA_CORE != 1) IF ( ( switch .EQ. input_only ) .OR. & ( switch .EQ. history_only ) .OR. & + ( ( switch .GE. auxhist1_only ) .AND. ( switch .LE. auxhist24_only ) ) .OR. & ( switch .EQ. restart_only ) ) THEN IF ( grid%this_is_an_ideal_run ) THEN CALL wrf_put_dom_ti_char ( fid , 'SIMULATION_INITIALIZATION_TYPE', "IDEALIZED DATA" , ierr ) From 0ca0d96d0f8640729149fa5cacfb7d79352cc989 Mon Sep 17 00:00:00 2001 From: weiwangncar Date: Thu, 29 Aug 2024 20:51:53 -0600 Subject: [PATCH 07/30] Add reflectivity calculation back to WSM6 scheme (#2098) TYPE: bug fix KEYWORDS: radar reflectivity, WSM6 SOURCE: reported by Gabriel Cassol of Sigma Meteorologia, Brazil, internal DESCRIPTION OF CHANGES: Problem: When moving WSM6 to MMM shared physics directory, the calculation of radar reflectivity was left out. Solution: The calculation of radar reflectivity is added back in the WSM6 scheme. ISSUE: For use when this PR closes an issue. Fixes https://github.com/wrf-model/WRF/issues/2096 LIST OF MODIFIED FILES: M phys/module_mp_wsm6.F TESTS CONDUCTED: 1. The reflectivity is computed now when namelist do_radar_ref is set to 1. 2. The Jenkins tests are all passing. RELEASE NOTE: Fixed the bug of missing radar reflectivity in WSM6 scheme when do_radar_ref is set to 1. This bug was introduced when WSM6 scheme was moved to physics_mmm/ directory, and shared with other MMM models in v4.6. --- phys/module_mp_wsm6.F | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/phys/module_mp_wsm6.F b/phys/module_mp_wsm6.F index d0d45e69a2..cf163d7a2f 100644 --- a/phys/module_mp_wsm6.F +++ b/phys/module_mp_wsm6.F @@ -1,7 +1,7 @@ module module_mp_wsm6 use ccpp_kind_types,only: kind_phys - use mp_wsm6,only: mp_wsm6_run + use mp_wsm6,only: mp_wsm6_run,refl10cm_wsm6 use mp_wsm6_effectrad,only: mp_wsm6_effectRad_run @@ -231,6 +231,23 @@ subroutine wsm6(th,q,qc,qr,qi,qs,qg,den,pii,p,delz, & enddo enddo + if (diagflag .and. do_radar_ref == 1) then + do i=its,ite + do k=kts,kte + t1d(k)=th(i,k,j)*pii(i,k,j) + p1d(k)=p(i,k,j) + qv1d(k)=q(i,k,j) + qr1d(k)=qr(i,k,j) + qs1d(k)=qs(i,k,j) + qg1d(k)=qg(i,k,j) + enddo + call refl10cm_wsm6 (qv1d,qr1d,qs1d,qg1d,t1d,p1d,dBZ,kts,kte) + do k = kts, kte + refl_10cm(i,k,j) = MAX(-35., dBZ(k)) + enddo + enddo + endif + enddo end subroutine wsm6 From a6056c1a1bbc2341e4d8607686d6bff49f916c83 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:16:48 -0700 Subject: [PATCH 08/30] Remove unsupported -auto C flag from Intel oneAPI stanza (#2059) TYPE: bug fix KEYWORDS: intel, compilation, oneapi SOURCE: internal DESCRIPTION OF CHANGES: Problem: The `-auto` flag is not supported by the Intel C compilers. See https://www.intel.com/content/www/us/en/docs/cpp-compiler/developer-guide-reference/2021-10/alphabetical-option-list.html for full list of supported icx flags. Solution: Remove the flag, no output changes should happen as the flag was previously being ignored if it didn't cause an error. --- arch/configure.defaults | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/configure.defaults b/arch/configure.defaults index 1275f3ce33..8874630057 100644 --- a/arch/configure.defaults +++ b/arch/configure.defaults @@ -2132,7 +2132,7 @@ DESCRIPTION = INTEL ($SFC/$SCC) : oneAPI LLVM DMPARALLEL = # 1 OMPCPP = # -D_OPENMP OMP = # -qopenmp -fpp -auto -OMPCC = # -qopenmp -auto +OMPCC = # -qopenmp SFC = ifx SCC = icx CCOMP = icx From 3dc9f57e0c16e6bdadb464cfc7db13e8a0cf290d Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Wed, 11 Sep 2024 15:19:16 -0700 Subject: [PATCH 09/30] Fix line continuation syntax causing problems with certain compilers (#2060) TYPE: bug fix KEYWORDS: syntax, compilation SOURCE: internal DESCRIPTION OF CHANGES: Problem: dyn_em/module_big_step_utilities_em.F contains nonstandard line continuation symbols that are most likely being ignored by the make build system by the in situ file preprocessing steps. The cmake build does not use sed or standard.exe to preprocess files, so these lines get fed directly to compilers. Fortran compilers that do not support C-style line continuation then fail (nvhpc as an example). Solution: Change line continuation characters to standard Fortran. --- dyn_em/module_big_step_utilities_em.F | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dyn_em/module_big_step_utilities_em.F b/dyn_em/module_big_step_utilities_em.F index 72e827b275..1d24d3cf00 100644 --- a/dyn_em/module_big_step_utilities_em.F +++ b/dyn_em/module_big_step_utilities_em.F @@ -5733,11 +5733,11 @@ SUBROUTINE moist_physics_finish_em( t_new, t_old, t0, mut, & ! 300 ! term 4 ! IF ( ( config_flags%use_theta_m .EQ. 1 ) .AND. (P_Qv .GE. PARAM_FIRST_SCALAR) ) THEN - t_new(i,k,j) = h_diabatic(i,k,j)*(1. + (R_v/R_d)*qv_diabatic(i,k,j)) + \ - mpten*(1. + (R_v/R_d)*qv(i,k,j)) + \ + t_new(i,k,j) = h_diabatic(i,k,j)*(1. + (R_v/R_d)*qv_diabatic(i,k,j)) + & + mpten*(1. + (R_v/R_d)*qv(i,k,j)) + & (R_v/R_d)*qvten*th_phy(i,k,j) - T0 th_phy_m_t0(i,k,j) = (t_new(i,k,j)+T0)/(1.+(R_v/R_d)*qv(i,k,j)) - T0 - h_diabatic(i,k,j) = ( mpten*(1. + (R_v/R_d)*qv(i,k,j)) + \ + h_diabatic(i,k,j) = ( mpten*(1. + (R_v/R_d)*qv(i,k,j)) + & (R_v/R_d)*qvten*th_phy(i,k,j) ) / dt ELSE t_new(i,k,j) = t_new(i,k,j) + mpten From d672f54ae9492ee7f47b9d4afbe0438c53b09f4e Mon Sep 17 00:00:00 2001 From: dudhia Date: Thu, 12 Sep 2024 16:58:49 -0600 Subject: [PATCH 10/30] Fix for possible undefined zol (#2105) TYPE: bug fix KEYWORDS: sfclayrev, fix possible undefined SOURCE: internal reported by Xin-Zhong Liang DESCRIPTION OF CHANGES: Problem: zol(i) can be left undefined in rare case of br(i) = 0. Solution: add line to initialize zol(i)=0. LIST OF MODIFIED FILES: M phys/physics_mmm/sf_sfclayrev.F90 TESTS CONDUCTED: 1. Only compile tested. 2. The Jenkins tests are all passing. RELEASE NOTE: Fix for rare undefined zol in sfclayrev. --- phys/physics_mmm/sf_sfclayrev.F90 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/phys/physics_mmm/sf_sfclayrev.F90 b/phys/physics_mmm/sf_sfclayrev.F90 index d05ff3e45a..f34701c57b 100644 --- a/phys/physics_mmm/sf_sfclayrev.F90 +++ b/phys/physics_mmm/sf_sfclayrev.F90 @@ -378,6 +378,8 @@ subroutine sf_sfclayrev_run(ux,vx,t1d,qv1d,p1d,dz8w1d, & do 320 i = its,ite ! + zol(i)=0. +! if(br(i).gt.0) then if(br(i).gt.250.0) then zol(i)=zolri(250.0,za(i),znt(i)) From 59d1ab12fb8dca8340cc037bdd8ebdd0dc0b0980 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Fri, 13 Sep 2024 12:34:43 -0700 Subject: [PATCH 11/30] Replace nonstandard isnan() call with value input checks instead (#2061) TYPE: bug fix KEYWORDS: compilation, nonstandard SOURCE: internal DESCRIPTION OF CHANGES: Problem: PR #1944 introduced a nonstandard function call `isnan()` which breaks compilation for compilers which don't support this extension. Solution: A potential alternative is to use the `ieee_is_nan()` call from `ieee_arithmetic`. As there is conditional compilation of this module which requires Fortran 2003 standard, an easier alternative is just to check the input values before calling `acos()` --- phys/module_wind_mav.F | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/phys/module_wind_mav.F b/phys/module_wind_mav.F index dabb6f1e36..ab779fda05 100644 --- a/phys/module_wind_mav.F +++ b/phys/module_wind_mav.F @@ -1230,10 +1230,11 @@ function find_turb(xc, yc, xt, yt, u, v, sr_angle, sr_dis) result(ft) tmp2 = sign(tmp1,tmp2) end if - angle = real(acos(tmp1/tmp2), kind = 4) - - if (isnan(angle)) then - angle = 0. + tmp1 = tmp1 / tmp2 + if (tmp1 < -1.0_8 .or. tmp1 > 1.0_8) then + angle = 0. + else + angle = real(acos(tmp1), kind = 4) end if if (abs(angle) <= sr_angle) then From 297b434784943433c4d1faf6b9af122d72630984 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:21:58 -0700 Subject: [PATCH 12/30] Add explicit casts to mismatched pointer types in rsl_lite (#2062) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TYPE: bug fix KEYWORDS: gcc14, compilation, c99 SOURCE: internal DESCRIPTION OF CHANGES: Problem: From #2047 : Compilation of WRF v4.6.0 fails with GCC 14 due to new standard changes > Type checking on pointer types (-Werror=incompatible-pointer-types) > GCC no longer allows implicitly casting all pointer types to all other pointer types. This behavior is now restricted to the void * type and its qualified variations. > > To fix compilation errors resulting from that, you can add the appropriate casts, and maybe consider using void * in more places (particularly for old programs that predate the introduction of void * into the C language). > > Programs that do not carefully track pointer types are likely to contain aliasing violations, so consider building with -fno-strict-aliasing. (Whether casts are written manually or performed by GCC automatically does not make a difference in terms of strict aliasing violations.) > > A frequent source of incompatible function pointer types involves callback functions that have more specific argument types (or less specific return types) than the function pointer they are assigned to. For example, old code which attempts to sort an array of strings might look like this: ``` #include #include #include int compare (char **a, char **b) { return strcmp (*a, *b); } void sort (char **array, size_t length) { qsort (array, length, sizeof (*array), compare); } ``` Example of failure : ``` /home/aislas/wrf-model/wrf/external/RSL_LITE/c_code.c: In function ‘rsl_lite_pack_’: /home/aislas/wrf-model/wrf/external/RSL_LITE/c_code.c:487:27: error: passing argument 1 of ‘f_pack_lint_’ from incompatible pointer type [-Wincompatible-pointer-types] 487 | F_PACK_LINT ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, | ^~~ | | | char * In file included from /home/aislas/wrf-model/wrf/external/RSL_LITE/c_code.c:31: /home/aislas/wrf-model/wrf/external/RSL_LITE/rsl_lite.h:200:25: note: expected ‘long int *’ but argument is of type ‘char *’ 200 | void F_PACK_LINT (long *inbuf, long *outbuf, int* memorder, int* js, int* je, int* ks, int* ke, int* is, int* ie, int* jms, int* jme, int* kms, int* kme, int* ims, int* ime, int* curs); | ~~~~~~^~~~~ /home/aislas/wrf-model/wrf/external/RSL_LITE/c_code.c:487:33: error: passing argument 2 of ‘f_pack_lint_’ from incompatible pointer type [-Wincompatible-pointer-types] 487 | F_PACK_LINT ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, | ~^~~~~~~~ | | | char * /home/aislas/wrf-model/wrf/external/RSL_LITE/rsl_lite.h:200:38: note: expected ‘long int *’ but argument is of type ‘char *’ 200 | void F_PACK_LINT (long *inbuf, long *outbuf, int* memorder, int* js, int* je, int* ks, int* ke, int* is, int* ie, int* jms, int* jme, int* kms, int* kme, int* ims, int* ime, int* curs); | ~~~~~~^~~~~~ /home/aislas/wrf-model/wrf/external/RSL_LITE/c_code.c:492:26: error: passing argument 1 of ‘f_pack_int_’ from incompatible pointer type [-Wincompatible-pointer-types] 492 | F_PACK_INT ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, | ^~~ | | | char * /home/aislas/wrf-model/wrf/external/RSL_LITE/rsl_lite.h:201:23: note: expected ‘int *’ but argument is of type ‘char *’ 201 | void F_PACK_INT (int *inbuf, int *outbuf, int* memorder, int* js, int* je, int* ks, int* ke, int* is, int* ie, int* jms, int* jme, int* kms, int* kme, int* ims, int* ime, int* curs); | ~~~~~^~~~~ ``` Error was reproduced AlmaLinux release 9.4 with GCC 14.1.0 Solution: Use explicit type casting ISSUE: Fixes #2047 TESTS CONDUCTED: 1. Tested on GCC 14.1.0 --- external/RSL_LITE/c_code.c | 64 ++++++++++++++++++------------------ external/RSL_LITE/period.c | 32 +++++++++--------- external/RSL_LITE/rsl_lite.h | 2 +- 3 files changed, 49 insertions(+), 49 deletions(-) diff --git a/external/RSL_LITE/c_code.c b/external/RSL_LITE/c_code.c index 8db9aba3ba..59e652d591 100755 --- a/external/RSL_LITE/c_code.c +++ b/external/RSL_LITE/c_code.c @@ -484,12 +484,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+yp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+yp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } @@ -505,12 +505,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; is = IMAX(ips-shw) ; ie = IMIN(ipe+shw) ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+yp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+yp_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+yp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+yp_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } @@ -538,12 +538,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+ym_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+ym_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+ym_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+ym_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } @@ -559,12 +559,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; is = IMAX(ips-shw) ; ie = IMIN(ipe+shw) ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+ym_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+ym_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+ym_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+ym_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } @@ -596,12 +596,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+xp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+xp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+xp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+xp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } @@ -617,12 +617,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; is = ipe+recvbegp ; ie = is + recvwp - 1 ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+xp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+xp_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+xp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+xp_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } @@ -652,12 +652,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+xm_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+xm_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+xm_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+xm_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } @@ -673,12 +673,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; ie = ips-recvbegm ; is = ie - recvwm + 1 ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+xm_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+xm_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+xm_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+xm_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } @@ -780,12 +780,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_UNPACK_LINT_AD ( p+yp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT_AD ( (long *)(p+yp_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT_AD ( p+yp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT_AD ( (int * )(p+yp_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } @@ -801,12 +801,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; is = IMAX(ips-shw) ; ie = IMIN(ipe+shw) ; if ( typesize == 8 ) { - F_PACK_LINT_AD ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT_AD ( (long *)buf, (long *)(p+yp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT_AD ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT_AD ( (int * )buf, (int * )(p+yp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } @@ -834,12 +834,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_UNPACK_LINT_AD ( p+ym_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT_AD ( (long *)(p+ym_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT_AD ( p+ym_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT_AD ( (int * )(p+ym_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } @@ -855,12 +855,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; is = IMAX(ips-shw) ; ie = IMIN(ipe+shw) ; if ( typesize == 8 ) { - F_PACK_LINT_AD ( buf, p+ym_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT_AD ( (long *)buf, (long *)(p+ym_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT_AD ( buf, p+ym_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT_AD ( (int * )buf, (int * )(p+ym_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } @@ -892,12 +892,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_UNPACK_LINT_AD ( p+xp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT_AD ( (long *)(p+xp_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT_AD ( p+xp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT_AD ( (int * )(p+xp_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } @@ -913,12 +913,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; is = ipe+recvbegp ; ie = is + recvwp - 1 ; if ( typesize == 8 ) { - F_PACK_LINT_AD ( buf, p+xp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT_AD ( (long *)buf, (long *)(p+xp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT_AD ( buf, p+xp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT_AD ( (int * )buf, (int * )(p+xp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } @@ -948,12 +948,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_UNPACK_LINT_AD ( p+xm_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT_AD ( (long *)(p+xm_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT_AD ( p+xm_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT_AD ( (int * )(p+xm_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } @@ -969,12 +969,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; ie = ips-recvbegm ; is = ie - recvwm + 1 ; if ( typesize == 8 ) { - F_PACK_LINT_AD ( buf, p+xm_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT_AD ( (long *)buf, (long *)(p+xm_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT_AD ( buf, p+xm_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT_AD ( (int * )buf, (int * )(p+xm_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } diff --git a/external/RSL_LITE/period.c b/external/RSL_LITE/period.c index b0401d03c5..e48ebd9e86 100755 --- a/external/RSL_LITE/period.c +++ b/external/RSL_LITE/period.c @@ -169,12 +169,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz MPI_Abort(MPI_COMM_WORLD, 98) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+xp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+xp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+xp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+xp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } @@ -188,12 +188,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz ks = kps ; ke = kpe ; is = ipe ; ie = ipe+shw-1+stag ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+xp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+xp_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+xp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+xp_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } @@ -219,12 +219,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz MPI_Abort(MPI_COMM_WORLD, 98) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+xm_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+xm_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+xm_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+xm_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } @@ -238,12 +238,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz ks = kps ; ke = kpe ; is = ips-shw ; ie = ips-1 ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+xm_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+xm_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+xm_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+xm_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } @@ -274,12 +274,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz MPI_Abort(MPI_COMM_WORLD, 98) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+yp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+yp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } @@ -293,12 +293,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz ks = kps ; ke = kpe ; js = jpe ; je = jpe+shw-1+stag ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+yp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+yp_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+yp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+yp_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } @@ -324,12 +324,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz MPI_Abort(MPI_COMM_WORLD, 98) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+ym_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+ym_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+ym_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+ym_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } @@ -343,12 +343,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz ks = kps ; ke = kpe ; js = jps-shw ; je = jps-1 ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+ym_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+ym_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+ym_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+ym_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } diff --git a/external/RSL_LITE/rsl_lite.h b/external/RSL_LITE/rsl_lite.h index f3e291350f..011401b493 100644 --- a/external/RSL_LITE/rsl_lite.h +++ b/external/RSL_LITE/rsl_lite.h @@ -161,7 +161,7 @@ #define RSL_MALLOC(T,N) (T *)rsl_malloc(__FILE__,__LINE__,(sizeof(T))*(N)) -#define RSL_FREE(P) rsl_free(&(P)) +#define RSL_FREE(P) rsl_free((char **)&(P)) char * buffer_for_proc ( int P, int size, int code ) ; int buffer_size_for_proc( int P, int code ); From 1d86bcb434a342226a0d76d5534793cabd0f88e5 Mon Sep 17 00:00:00 2001 From: Jesus Fernandez Date: Wed, 18 Sep 2024 16:40:20 +0200 Subject: [PATCH 13/30] Fix non-initialized mixing ratio in p/z-lev interp (#2104) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TYPE: bug fix KEYWORDS: mixing ratio, interpolation, pressure level, height level, missing value SOURCE: Jesus Fernandez, Andrés Simón-Moral and Josipa Milovac (Instituto de Física de Cantabria, CSIC-Universidad de Cantabria, Spain) DESCRIPTION OF CHANGES: Problem: The mixing ratio field is not being initialized in the interpolation routine. This leads to no missing values below ground and values from previous time steps filling these areas. Solution: The mixing ratio has been initialized to missing values, as for the rest of variables. LIST OF MODIFIED FILES: ``` M phys/module_diag_pld.F M phys/module_diag_zld.F ``` TESTS CONDUCTED: The Jenkins tests are all passing. RELEASE NOTE: Fix non-initialized 3D moisture field in pressure- and height-level diagnostic interpolation. --- phys/module_diag_pld.F | 1 + phys/module_diag_zld.F | 1 + 2 files changed, 2 insertions(+) diff --git a/phys/module_diag_pld.F b/phys/module_diag_pld.F index 88fe6aa486..cba25e0a59 100644 --- a/phys/module_diag_pld.F +++ b/phys/module_diag_pld.F @@ -82,6 +82,7 @@ SUBROUTINE pld ( u,v,w,t,qv,zp,zb,pp,pb,p,pw, & ght_pl(i,kp,j) = missing s_pl (i,kp,j) = missing td_pl (i,kp,j) = missing + q_pl (i,kp,j) = missing END DO END DO END DO diff --git a/phys/module_diag_zld.F b/phys/module_diag_zld.F index 6156ecbb0e..a0d5eacc39 100644 --- a/phys/module_diag_zld.F +++ b/phys/module_diag_zld.F @@ -82,6 +82,7 @@ SUBROUTINE zld ( u,v,w,t,qv,zp,zb,pp,pb,p,pw, & ght_zl(i,kz,j) = missing s_zl (i,kz,j) = missing td_zl (i,kz,j) = missing + q_zl (i,kz,j) = missing p_zl (i,kz,j) = missing END DO END DO From 958ce123c548645e56db9a2d6b143a69c75536c1 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Thu, 19 Sep 2024 16:41:30 -0700 Subject: [PATCH 14/30] Initial testing framework (#2095) TYPE: enhancement KEYWORDS: testing, regression, test framework SOURCE: internal DESCRIPTION OF CHANGES: Problem: The current regression suite code is complex, requires maintenance of multiple alternate repositories, and takes involved effort to add a new test making community contribution limited at best. Likewise, the complexity of the system reduces the likelihood of independent local testing of changes, leading to a development cycle of one-off commits done to reinvoke testing to see if meaningful commits fix the issues. Solution: This new proposed regression suite addresses these shortcomings in a number of discrete ways: 1. Modularize the testing framework to an generalized independent repo usable by any repo seeking to set up tests that can run locally, on HPC systems, and within any CI/CD framework 2. Write WRF-specific test scripts _inside_ the WRF repo and in a manner that does not rely on specific layouts/hardware/etc. so long as WRF can compile and run on intended system (i.e. able to be run locally) 3. Write CI/CD tests in a simple and generally CI/CD framework-agnostic method where definitions of these also reside _within the WRF repo_ 4. Utilize HPC resources in a safe manner to increase breadth of testing to allow testing of many more compilers and on similar hardware to the general use case of WRF As a first pass at demonstrating this solution, this PR implements a simple set of compilation tests using GNU x86 configurations testing serial, sm, dm, and sm+dm selections. The CI/CD portion is done via GitHub workflow actions on a specific trigger event. The values and trigger methods are configurable, but this initial implementation will use the `labeled` trigger, which will initiate tests when `compile-tests` or `all-tests` is added as a label to a pull request. TESTS CONDUCTED: 1. Testing of this github workflow was done in a separate fork also testing on Derecho. Both positive and negative tests were used to demonstrate respective output usefulness. RELEASE NOTE: Introduce a modularized testing framework that allows testing locally and natively on HPC systems that lives within the WRF repository --- .ci/env/derecho.sh | 22 ++++ .ci/env/helpers.sh | 46 +++++++++ .ci/env/hostenv.sh | 16 +++ .ci/hpc-workflows | 1 + .ci/tests/build.sh | 108 ++++++++++++++++++++ .ci/wrf_compilation_tests-make.json | 69 +++++++++++++ .github/workflows/ci.yml | 97 ++++++++++++++++++ .github/workflows/test_workflow.yml | 150 ++++++++++++++++++++++++++++ .gitignore | 1 + .gitmodules | 3 + 10 files changed, 513 insertions(+) create mode 100755 .ci/env/derecho.sh create mode 100644 .ci/env/helpers.sh create mode 100644 .ci/env/hostenv.sh create mode 160000 .ci/hpc-workflows create mode 100755 .ci/tests/build.sh create mode 100644 .ci/wrf_compilation_tests-make.json create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/test_workflow.yml diff --git a/.ci/env/derecho.sh b/.ci/env/derecho.sh new file mode 100755 index 0000000000..2feea5c9ee --- /dev/null +++ b/.ci/env/derecho.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +echo "Setting up derecho environment" +workingDirectory=$PWD +. /etc/profile.d/z00_modules.sh +echo "Loading modules : $*" +cmd="module purge" +echo $cmd && eval "${cmd}" + +# We should be handed in the modules to load +while [ $# -gt 0 ]; do + cmd="module load $1" + echo $cmd && eval "${cmd}" + shift +done + +# Go back to working directory if for unknown reason HPC config changing your directory on you +if [ "$workingDirectory" != "$PWD" ]; then + echo "derecho module loading changed working directory" + echo " Moving back to $workingDirectory" + cd $workingDirectory +fi diff --git a/.ci/env/helpers.sh b/.ci/env/helpers.sh new file mode 100644 index 0000000000..2ddd560893 --- /dev/null +++ b/.ci/env/helpers.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# Useful string manipulation functions, leaving in for posterity +# https://stackoverflow.com/a/8811800 +# contains(string, substring) +# +# Returns 0 if the specified string contains the specified substring, +# otherwise returns 1. +contains() +{ + string="$1" + substring="$2" + + if [ "${string#*"$substring"}" != "$string" ]; then + echo 0 # $substring is in $string + else + echo 1 # $substring is not in $string + fi +} + +setenvStr() +{ + # Changing IFS produces the most consistent results + tmpIFS=$IFS + IFS="," + string="$1" + for s in $string; do + if [ ! -z $s ]; then + eval "echo export \"$s\"" + eval "export \"$s\"" + fi + done + IFS=$tmpIFS +} + +banner() +{ + lengthBanner=$1 + shift + # https://www.shellscript.sh/examples/banner/ + printf "#%${lengthBanner}s#\n" | tr " " "=" + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" "`date`" + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" " " + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" "$*" + printf "#%${lengthBanner}s#\n" | tr " " "=" +} diff --git a/.ci/env/hostenv.sh b/.ci/env/hostenv.sh new file mode 100644 index 0000000000..208d1e57f3 --- /dev/null +++ b/.ci/env/hostenv.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# Allow selection of hostname, and if none is provided use the current machine +# While this may seem unintuitive at first, it provides the flexibility of using +# "named" configurations without being explicitly tied to fqdn +hostname=$AS_HOST +if [ -z "$hostname" ]; then + hostname=$( python3 -c "import socket; print( socket.getfqdn() )" ) +fi + +if [ $( contains ${hostname} hsn.de.hpc ) -eq 0 ]; then + # Derecho HPC SuSE PBS + . .ci/env/derecho.sh +else + echo "No known environment for '${hostname}', using current" +fi diff --git a/.ci/hpc-workflows b/.ci/hpc-workflows new file mode 160000 index 0000000000..ba8393447c --- /dev/null +++ b/.ci/hpc-workflows @@ -0,0 +1 @@ +Subproject commit ba8393447c8a2cef23952c01425154ceb34d64e4 diff --git a/.ci/tests/build.sh b/.ci/tests/build.sh new file mode 100755 index 0000000000..a0598fd499 --- /dev/null +++ b/.ci/tests/build.sh @@ -0,0 +1,108 @@ +#!/bin/sh +help() +{ + echo "./build.sh as_host workingdir [options] [-- ]" + echo " as_host First argument must be the host configuration to use for environment loading" + echo " workingdir First argument must be the working dir to immediate cd to" + echo " -c Configuration build type, piped directly into configure" + echo " -n Configuration nesting type, piped directly into configure" + echo " -o Configuration optstring passed into configure" + echo " -b Build command passed into compile" + echo " -e environment variables in comma-delimited list, e.g. var=1,foo,bar=0" + echo " -- Directly pass options to hostenv.sh, equivalent to hostenv.sh " + echo " -h Print this message" + echo "" + echo "If you wish to use an env var in your arg such as '-c \$SERIAL -e SERIAL=32', you must" + echo "you will need to do '-c \\\$SERIAL -e SERIAL=32' to delay shell expansion" +} + +echo "Input arguments:" +echo "$*" + +AS_HOST=$1 +shift +if [ $AS_HOST = "-h" ]; then + help + exit 0 +fi + +workingDirectory=$1 +shift + +cd $workingDirectory + +# Get some helper functions +. .ci/env/helpers.sh + +while getopts c:n:o:b:e:h opt; do + case $opt in + c) + configuration="$OPTARG" + ;; + n) + nesting="$OPTARG" + ;; + o) + configOpt="$OPTARG" + ;; + b) + buildCommand="$OPTARG" + ;; + e) + envVars="$envVars,$OPTARG" + ;; + h) help; exit 0 ;; + *) help; exit 1 ;; + :) help; exit 1 ;; + \?) help; exit 1 ;; + esac +done + +shift "$((OPTIND - 1))" + +# Everything else goes to our env setup +. .ci/env/hostenv.sh $* + +# Now evaluate env vars in case it pulls from hostenv.sh +if [ ! -z "$envVars" ]; then + setenvStr "$envVars" +fi + +# Re-evaluate input values for delayed expansion +eval "configuration=\"$configuration\"" +eval "nesting=\"$nesting\"" +eval "configOpt=\"$configOpt\"" +eval "buildCommand=\"$buildCommand\"" + +./clean -a + +echo "Compiling with option $configuration nesting=$nesting and additional flags '$configOpt'" +./configure $configOpt << EOF +$configuration +$nesting +EOF + +if [ ! -f configure.wrf ]; then + echo "Failed to configure" + exit 1 +fi + +echo "./compile $buildCommand" +./compile $buildCommand + +result=$? + +if [ $result -ne 0 ]; then + echo "Failed to compile" + exit 1 +fi + +# And a *very* special check because WRF compiles the WRF way and force-ignores all make errors +# putting the onus on US to check for things +if [ ! -x ./main/wrf.exe ]; then # There's a bunch of other execs but this is the most important and + # doing more checks to accomodate just reinforces this bad design + echo "Failed to compile" + exit 1 +fi + +echo "TEST $(basename $0) PASS" diff --git a/.ci/wrf_compilation_tests-make.json b/.ci/wrf_compilation_tests-make.json new file mode 100644 index 0000000000..b80a2b8b53 --- /dev/null +++ b/.ci/wrf_compilation_tests-make.json @@ -0,0 +1,69 @@ +{ + "submit_options" : + { + "timelimit" : "00:20:00", + "working_directory" : "..", + "arguments" : + { + "base_env_numprocs" : [ "-e", "NUM_PROCS=4" ], + + ".*make.*::args_nesting" : [ "-n", "1" ], + ".*make.*::args_configopt" : [ "-o", "-d" ], + ".*make.*::args_build_tgt" : [ "-b", "em_real -j $NUM_PROCS" ] + }, + "hsn.de.hpc" : + { + "submission" : "PBS", + "queue" : "main", + "hpc_arguments" : + { + "node_select" : { "-l " : { "select" : 1, "ncpus" : 16 } }, + "priority" : { "-l " : { "job_priority" : "economy" } } + }, + "arguments" : + { + "base_env_numprocs" : [ "-e", "NUM_PROCS=16" ], + "very_last_modules" : [ "netcdf" ], + ".*gnu.*::test_modules" : [ "gcc" ], + ".*intel(?!-llvm).*::test_modules" : [ "intel-classic" ], + ".*intel-llvm.*::test_modules" : [ "intel-oneapi" ], + ".*pgi.*::test_modules" : [ "nvhpc" ], + ".*dm.*::test_mpi_module" : [ "cray-mpich" ] + } + } + }, + "make-gnu" : + { + "steps" : + { + "serial" : + { + "command" : ".ci/tests/build.sh", + "arguments" : [ "-c", "32" ] + }, + "sm" : + { + "command" : ".ci/tests/build.sh", + "arguments" : [ "-c", "33" ], + "dependencies" : { "serial" : "afterany" } + } + } + }, + "make-gnu-mpi" : + { + "steps" : + { + "dm" : + { + "command" : ".ci/tests/build.sh", + "arguments" : [ "-c", "34" ] + }, + "dm+sm" : + { + "command" : ".ci/tests/build.sh", + "arguments" : [ "-c", "35" ], + "dependencies" : { "dm" : "afterany" } + } + } + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..ec396e2ce2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,97 @@ +name: Regression Suite +run-name : ${{ github.event_name == 'push' && 'CI' || github.event.label.name }} (${{ github.event_name }}) + +on: + push: + branches: [ master, develop ] +# See https://stackoverflow.com/a/78444521 and +# https://github.com/orgs/community/discussions/26874#discussioncomment-3253755 +# as well as official (but buried) documentation : +# https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull-request-events-for-forked-repositories-2 + pull_request: + types: [ labeled ] + +# https://docs.github.com/en/actions/sharing-automations/reusing-workflows#supported-keywords-for-jobs-that-call-a-reusable-workflow +# Also https://stackoverflow.com/a/74959635 +# TL;DR - For public repositories the safest approach will be to use the default read permissions, but at the cost +# of not being able to modify the labels. That will need to be a separate [trusted] workflow that runs from the base repo +# permissions : +# contents : read +# pull-requests : write + +# Write our tests out this way for easier legibility +# testsSet : +# - key : value +# key : value +# tests : +# - value +# - value +# - < next test > +# https://stackoverflow.com/a/68940067 +jobs: + buildtests: + if : ${{ github.event.label.name == 'compile-tests' || github.event.label.name == 'all-tests' || github.event_name == 'push' }} + strategy: + max-parallel: 4 + fail-fast: false + matrix: + + testSet : + - host : derecho + hpc-workflows_path : .ci/hpc-workflows + archive : /glade/work/aislas/github/runners/wrf/derecho/logs/ + account : NMMM0012 + name : "Make Compilation Tests" + id : make-tests + fileroot : wrf_compilation_tests-make + args : -j='{"node_select":{"-l ":{"select":1}}}' + pool : 8 + tpool : 1 + mkdirs : true + tests : + - make-gnu + - make-gnu-mpi + # add new compilation tests here + + uses : ./.github/workflows/test_workflow.yml + with : + # This should be the only hard-coded value, we don't use ${{ github.event.label.name }} + # to avoid 'all-tests' to be used in this workflow + label : compile-tests + + # Everything below this should remain the same and comes from the testSet matrix + hpc-workflows_path : ${{ matrix.testSet.hpc-workflows_path }} + archive : ${{ matrix.testSet.archive }} + name : ${{ matrix.testSet.name }} + id : ${{ matrix.testSet.id }} + host : ${{ matrix.testSet.host }} + fileroot : ${{ matrix.testSet.fileroot }} + account : ${{ matrix.testSet.account }} + tests : ${{ toJson( matrix.testSet.tests ) }} + mkdirs : ${{ matrix.testSet.mkdirs }} + args : ${{ matrix.testSet.args }} + pool : ${{ matrix.testSet.pool }} + tpool : ${{ matrix.testSet.tpool }} + # I am leaving this here for posterity if this is to be replicated in private repositories for testing + permissions: + contents: read + pull-requests: write + name : Test ${{ matrix.testSet.name }} on ${{ matrix.testSet.host }} + + # In the event that 'all-tests' is used, this final job will be the one to remove + # the label from the PR + removeAllLabel : + if : ${{ !cancelled() && github.event.label.name == 'all-tests' }} + name : Remove 'all-tests' label + runs-on: ubuntu-latest + needs : [ buildtests ] # Put tests here to make this wait for the tests to complete + steps: + - name : Remove '${{ github.event.label.name }}' label + env: + PR_NUMBER: ${{ github.event.number }} + run: | + curl \ + -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + -H 'Authorization: token ${{ github.token }}' \ + https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ github.event.label.name }} diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml new file mode 100644 index 0000000000..abcb901c0a --- /dev/null +++ b/.github/workflows/test_workflow.yml @@ -0,0 +1,150 @@ + + +on : + workflow_call : + inputs : + label : + required : true + type : string + hpc-workflows_path : + required : true + type : string + archive : + required : true + type : string + + name : + required : true + type : string + id : + required : true + type : string + host : + required : true + type : string + fileroot : + required : true + type : string + account : + required : true + type : string + tests : + required : true + type : string + mkdirs : + required : true + type : boolean + args : + required : false + type : string + default : "" + pool : + required : false + type : number + default : 1 + tpool : + required : false + type : number + default : 1 + + + +jobs: + test_workflow : + + # Is 5 days a reasonable wait time for testing? + timeout-minutes: 7200 + name: Test ${{ inputs.name }} on ${{ inputs.host }} + runs-on: ${{ inputs.host }} + env : + LOG_SUFFIX : ${{ github.event_name == 'push' && 'master' || github.event.number }} + steps: + - uses: actions/checkout@v4 + with: + path : main + submodules: true + + # Immediately copy out to # of tests to do + - name: Create testing directories + if : ${{ inputs.mkdirs }} + id : cpTestDirs + run : | + for testDir in ${{ join( fromJson( inputs.tests ), ' ' ) }}; do + echo "Creating duplicate directory for $testDir" + # Remove if it exists to get a fresh start + rm -rf $testDir + cp -Rp main/ $testDir + done + + - name: Test ${{ inputs.name }} + id : runTest + run: | + if [ "${{ inputs.mkdirs }}" = "true" ]; then + ALT_DIRS="-alt ../${{ join( fromJson( inputs.tests ), '/.ci ../' ) }}/.ci" + fi + ./main/${{ inputs.hpc-workflows_path }}/.ci/runner.py \ + ./main/.ci/${{ inputs.fileroot }}.json \ + -t ${{ join( fromJson( inputs.tests ), ' ' ) }} \ + -a "${{ inputs.account }}" \ + -p ${{ inputs.pool}} -tp ${{ inputs.tpool }} \ + ${{ inputs.args }} $ALT_DIRS + + + - name: Report failed tests and steps + if : ${{ failure() }} + run : | + # move log files to safe location + ./main/${{ inputs.hpc-workflows_path }}/.ci/relocator.py ./main/.ci/${{ inputs.fileroot }}.log ${{ inputs.archive }}/$LOG_SUFFIX/${{ inputs.id }} + + # report on them - alt dirs need extra help + if [ "${{ inputs.mkdirs }}" = "true" ]; then + masterlogLoc=main/.ci + fi + ./main/${{ inputs.hpc-workflows_path }}/.ci/reporter.py ${{ inputs.archive }}/$LOG_SUFFIX/${{ inputs.id }}/$masterlogLoc/${{ inputs.fileroot }}.log \ + -e ./${{ inputs.hpc-workflows_path }}/.ci/runner.py \ + -o GITHUB -m # only mark fail steps with gh syntax + + # report on them + echo "# Summary for ${{ join( fromJson( inputs.tests ), ' ' ) }}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + ./main/${{ inputs.hpc-workflows_path }}/.ci/reporter.py ${{ inputs.archive }}/$LOG_SUFFIX/${{ inputs.id }}/$masterlogLoc/${{ inputs.fileroot }}.log \ + -e ./${{ inputs.hpc-workflows_path }}/.ci/runner.py \ + -s >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + + - name: Clean up testing directories + if : ${{ success() }} + id : rmTestDirs + run : | + for testDir in ${{ join( fromJson( inputs.tests ), ' ' ) }}; do + echo "Removing duplicate directory for $testDir" + rm -rf $testDir + done + + - name: Upload test logs + if : ${{ failure() }} + uses : actions/upload-artifact@v4 + with: + # as per usual with ci/cd stuff I am shocked but not surprised when the advertised + # *documented* functionality doesn't work as expected. Wow, bravo + # can't use ${{ env. }} as somehow this combination of matrix->reusable workflow->call step is too complex + # and expands to nothing + name: ${{ github.event_name == 'push' && 'master' || github.event.number }}-${{ inputs.id }}_logfiles + path: ${{ inputs.archive }}/${{ github.event_name == 'push' && 'master' || github.event.number }}/${{ inputs.id }}/ + + # As noted in ci.yml, this will need to be moved to a separate workflow with pull_request_target + # and strictly controlled usage of the GH token + # - name : Remove '${{ inputs.label }}' label + # if : ${{ !cancelled() && github.event.label.name == inputs.label }} + # env: + # PR_NUMBER: ${{ github.event.number }} + # run: | + # curl \ + # -X DELETE \ + # -H "Accept: application/vnd.github.v3+json" \ + # -H 'Authorization: token ${{ github.token }}' \ + # https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ inputs.label }} + + + + diff --git a/.gitignore b/.gitignore index 876fb30491..09e62c68a5 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ wrfout_d* *.nc rsl.out.* rsl.error.* +*.log diff --git a/.gitmodules b/.gitmodules index 4e323edda7..5d206bcbf5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "phys/noahmp"] path = phys/noahmp url = https://github.com/NCAR/noahmp +[submodule ".ci/hpc-workflows"] + path = .ci/hpc-workflows + url = https://github.com/islas/hpc-workflows From ed585bda6457604d1867d29be8b4e9abaf649313 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:20:17 -0700 Subject: [PATCH 15/30] Restructure alloc space calls into simpler smaller files (#2068) TYPE: enhancement KEYWORDS: intel, compilation, llvm, memory SOURCE: internal DESCRIPTION OF CHANGES: Problem: The Intel oneAPI compilers (and others like nvhpc) struggle with some of the larger (15k+ lines of code) files within WRF. This causes intense memory usage that is not often available to the average user not in a resource-rich environment. This often limits compilation to single threaded if even possible or to a dedicated environment with enough memory if available. If neither of those is available to a user, they will be unable to use these configurations entirely. Solution: This PR focuses on the `module_alloc_space*` sections of code to reduce their individual file size to manageable levels. They are instead broken out into many smaller files as external subroutines, not requiring a wrapper module to house the subroutine call. The files are now fully generated source code from the registry, with the calls to the subroutines also being generated as well. This also makes it relatively easy to change the number of files generated from a source code perspective. Build rules would need to be modified accordingly as seen in these changes. TESTS CONDUCTED: Attached to this PR are plots of the respective effects of theses changes. Changes were tested with intel and gcc compilers, but only intel memory usage is shown as it exacerbates the memory usage issue. --- arch/noopt_exceptions | 21 --- arch/noopt_exceptions_f | 21 --- clean | 2 +- frame/CMakeLists.txt | 10 -- frame/Makefile | 42 ++++-- frame/module_alloc_space.h | 154 -------------------- frame/module_alloc_space_0.F | 19 --- frame/module_alloc_space_1.F | 19 --- frame/module_alloc_space_2.F | 19 --- frame/module_alloc_space_3.F | 19 --- frame/module_alloc_space_4.F | 19 --- frame/module_alloc_space_5.F | 19 --- frame/module_alloc_space_6.F | 19 --- frame/module_alloc_space_7.F | 19 --- frame/module_alloc_space_8.F | 19 --- frame/module_alloc_space_9.F | 19 --- frame/module_domain.F | 96 +------------ main/depend.common | 260 +++++++++++++++++++++++++++++---- tools/CMakeLists.txt | 20 +++ tools/gen_allocs.c | 272 ++++++++++++++++++++++++++++++++--- tools/protos.h | 2 +- var/build/da.make | 42 ++++-- var/build/da_name_space.pl | 2 +- var/build/depend.txt | 175 ++++++++++++++++++++-- 24 files changed, 737 insertions(+), 572 deletions(-) delete mode 100644 frame/module_alloc_space.h delete mode 100644 frame/module_alloc_space_0.F delete mode 100644 frame/module_alloc_space_1.F delete mode 100644 frame/module_alloc_space_2.F delete mode 100644 frame/module_alloc_space_3.F delete mode 100644 frame/module_alloc_space_4.F delete mode 100644 frame/module_alloc_space_5.F delete mode 100644 frame/module_alloc_space_6.F delete mode 100644 frame/module_alloc_space_7.F delete mode 100644 frame/module_alloc_space_8.F delete mode 100644 frame/module_alloc_space_9.F diff --git a/arch/noopt_exceptions b/arch/noopt_exceptions index f4fad7973a..b4826c05f0 100644 --- a/arch/noopt_exceptions +++ b/arch/noopt_exceptions @@ -17,16 +17,6 @@ module_comm_nesting_dm.o : module_comm_nesting_dm.F module_configure.o : module_configure.F module_domain.o : module_domain.F module_domain_type.o : module_domain_type.F -module_alloc_space_0.o : module_alloc_space_0.F -module_alloc_space_1.o : module_alloc_space_1.F -module_alloc_space_2.o : module_alloc_space_2.F -module_alloc_space_3.o : module_alloc_space_3.F -module_alloc_space_4.o : module_alloc_space_4.F -module_alloc_space_5.o : module_alloc_space_5.F -module_alloc_space_6.o : module_alloc_space_6.F -module_alloc_space_7.o : module_alloc_space_7.F -module_alloc_space_8.o : module_alloc_space_8.F -module_alloc_space_9.o : module_alloc_space_9.F module_tiles.o : module_tiles.F module_initialize.o : module_initialize.F module_physics_init.o : module_physics_init.F @@ -209,17 +199,6 @@ wrf_inputout.o \ wrf_restartin.o \ wrf_restartout.o \ module_state_description.o \ -module_alloc_space.o \ -module_alloc_space_0.o \ -module_alloc_space_1.o \ -module_alloc_space_2.o \ -module_alloc_space_3.o \ -module_alloc_space_4.o \ -module_alloc_space_5.o \ -module_alloc_space_6.o \ -module_alloc_space_7.o \ -module_alloc_space_8.o \ -module_alloc_space_9.o \ module_comm_dm.o \ module_comm_dm_0.o \ module_comm_dm_1.o \ diff --git a/arch/noopt_exceptions_f b/arch/noopt_exceptions_f index 3c3a0584e2..454eb2afc4 100644 --- a/arch/noopt_exceptions_f +++ b/arch/noopt_exceptions_f @@ -21,16 +21,6 @@ module_comm_nesting_dm.o : module_comm_nesting_dm.F module_configure.o : module_configure.F module_domain.o : module_domain.F module_domain_type.o : module_domain_type.F -module_alloc_space_0.o : module_alloc_space_0.F -module_alloc_space_1.o : module_alloc_space_1.F -module_alloc_space_2.o : module_alloc_space_2.F -module_alloc_space_3.o : module_alloc_space_3.F -module_alloc_space_4.o : module_alloc_space_4.F -module_alloc_space_5.o : module_alloc_space_5.F -module_alloc_space_6.o : module_alloc_space_6.F -module_alloc_space_7.o : module_alloc_space_7.F -module_alloc_space_8.o : module_alloc_space_8.F -module_alloc_space_9.o : module_alloc_space_9.F module_tiles.o : module_tiles.F module_initialize.o : module_initialize.F module_physics_init.o : module_physics_init.F @@ -124,17 +114,6 @@ wrf_inputout.o \ wrf_restartin.o \ wrf_restartout.o \ module_state_description.o \ -module_alloc_space.o \ -module_alloc_space_0.o \ -module_alloc_space_1.o \ -module_alloc_space_2.o \ -module_alloc_space_3.o \ -module_alloc_space_4.o \ -module_alloc_space_5.o \ -module_alloc_space_6.o \ -module_alloc_space_7.o \ -module_alloc_space_8.o \ -module_alloc_space_9.o \ module_comm_dm.o \ module_comm_dm_0.o \ module_comm_dm_1.o \ diff --git a/clean b/clean index 2fd6453194..85079ddc44 100755 --- a/clean +++ b/clean @@ -21,7 +21,7 @@ endif ( cd tools/CodeBase ; make clean ) -( cd inc ; /bin/rm -f *.inc namelist.default commit_decl ) +( cd inc ; /bin/rm -f *.inc namelist.default commit_decl *.o *.F ) find . -name \*.dSYM -exec rm -rf {} \; >& /dev/null diff --git a/frame/CMakeLists.txt b/frame/CMakeLists.txt index 59f8d2551b..2882d036ed 100644 --- a/frame/CMakeLists.txt +++ b/frame/CMakeLists.txt @@ -109,16 +109,6 @@ target_sources( module_cpl_oasis3.F - module_alloc_space_0.F - module_alloc_space_1.F - module_alloc_space_2.F - module_alloc_space_3.F - module_alloc_space_4.F - module_alloc_space_5.F - module_alloc_space_6.F - module_alloc_space_7.F - module_alloc_space_8.F - module_alloc_space_9.F ${CMAKE_BINARY_DIR}/frame/module_state_description.F # GENERATED ${nl_dyn_source} # GENERATED diff --git a/frame/Makefile b/frame/Makefile index a861a9e979..75973c87d3 100644 --- a/frame/Makefile +++ b/frame/Makefile @@ -35,16 +35,38 @@ MODULES = module_driver_constants.o \ module_cpl_oasis3.o \ module_clear_halos.o ALOBJS =\ - module_alloc_space_0.o \ - module_alloc_space_1.o \ - module_alloc_space_2.o \ - module_alloc_space_3.o \ - module_alloc_space_4.o \ - module_alloc_space_5.o \ - module_alloc_space_6.o \ - module_alloc_space_7.o \ - module_alloc_space_8.o \ - module_alloc_space_9.o + ../inc/allocs_0.o \ + ../inc/allocs_1.o \ + ../inc/allocs_2.o \ + ../inc/allocs_3.o \ + ../inc/allocs_4.o \ + ../inc/allocs_5.o \ + ../inc/allocs_6.o \ + ../inc/allocs_7.o \ + ../inc/allocs_8.o \ + ../inc/allocs_9.o \ + ../inc/allocs_10.o \ + ../inc/allocs_11.o \ + ../inc/allocs_12.o \ + ../inc/allocs_13.o \ + ../inc/allocs_14.o \ + ../inc/allocs_15.o \ + ../inc/allocs_16.o \ + ../inc/allocs_17.o \ + ../inc/allocs_18.o \ + ../inc/allocs_19.o \ + ../inc/allocs_20.o \ + ../inc/allocs_21.o \ + ../inc/allocs_22.o \ + ../inc/allocs_23.o \ + ../inc/allocs_24.o \ + ../inc/allocs_25.o \ + ../inc/allocs_26.o \ + ../inc/allocs_27.o \ + ../inc/allocs_28.o \ + ../inc/allocs_29.o \ + ../inc/allocs_30.o \ + ../inc/allocs_31.o NLOBJS =\ nl_get_0_routines.o \ diff --git a/frame/module_alloc_space.h b/frame/module_alloc_space.h deleted file mode 100644 index 56e8bf2f87..0000000000 --- a/frame/module_alloc_space.h +++ /dev/null @@ -1,154 +0,0 @@ - SUBROUTINE ROUTINENAME ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - - USE module_domain_type - USE module_configure, ONLY : model_config_rec, grid_config_rec_type, in_use_for_config, model_to_grid_config_rec -! USE module_state_description - USE module_scalar_tables ! this includes module_state_description too - - IMPLICIT NONE - - ! Input data. - - TYPE(domain) , POINTER :: grid - INTEGER , INTENT(IN) :: id - INTEGER , INTENT(IN) :: setinitval_in ! 3 = everything, 1 = arrays only, 0 = none - INTEGER , INTENT(IN) :: sd31, ed31, sd32, ed32, sd33, ed33 - INTEGER , INTENT(IN) :: sm31, em31, sm32, em32, sm33, em33 - INTEGER , INTENT(IN) :: sp31, ep31, sp32, ep32, sp33, ep33 - INTEGER , INTENT(IN) :: sp31x, ep31x, sp32x, ep32x, sp33x, ep33x - INTEGER , INTENT(IN) :: sp31y, ep31y, sp32y, ep32y, sp33y, ep33y - INTEGER , INTENT(IN) :: sm31x, em31x, sm32x, em32x, sm33x, em33x - INTEGER , INTENT(IN) :: sm31y, em31y, sm32y, em32y, sm33y, em33y - - ! this argument is a bitmask. First bit is time level 1, second is time level 2, and so on. - ! e.g. to set both 1st and second time level, use 3 - ! to set only 1st use 1 - ! to set only 2st use 2 - INTEGER , INTENT(IN) :: tl_in - - ! true if the allocation is for an intermediate domain (for nesting); only certain fields allocated - ! false otherwise (all allocated, modulo tl above) - LOGICAL , INTENT(IN) :: inter_domain_in, okay_to_alloc_in - - INTEGER(KIND=8) , INTENT(INOUT) :: num_bytes_allocated - - - ! Local data. - INTEGER idum1, idum2, spec_bdy_width - REAL initial_data_value - CHARACTER (LEN=256) message - INTEGER tl - LOGICAL inter_domain, okay_to_alloc - INTEGER setinitval - INTEGER sr_x, sr_y - - !declare ierr variable for error checking ALLOCATE calls - INTEGER ierr - - INTEGER :: loop - INTEGER(KIND=8) :: nba ! number of bytes allocated per variable - CHARACTER(LEN=256) :: message_string - - ! Local data - - TYPE ( grid_config_rec_type ) :: config_flags - - INTEGER :: k_start , k_end, its, ite, jts, jte - INTEGER :: ids , ide , jds , jde , kds , kde , & - ims , ime , jms , jme , kms , kme , & - ips , ipe , jps , jpe , kps , kpe - - INTEGER :: sids , side , sjds , sjde , skds , skde , & - sims , sime , sjms , sjme , skms , skme , & - sips , sipe , sjps , sjpe , skps , skpe - - - INTEGER :: imsx, imex, jmsx, jmex, kmsx, kmex, & - ipsx, ipex, jpsx, jpex, kpsx, kpex, & - imsy, imey, jmsy, jmey, kmsy, kmey, & - ipsy, ipey, jpsy, jpey, kpsy, kpey - - data_ordering : SELECT CASE ( model_data_order ) - CASE ( DATA_ORDER_XYZ ) - ids = sd31 ; ide = ed31 ; jds = sd32 ; jde = ed32 ; kds = sd33 ; kde = ed33 ; - ims = sm31 ; ime = em31 ; jms = sm32 ; jme = em32 ; kms = sm33 ; kme = em33 ; - ips = sp31 ; ipe = ep31 ; jps = sp32 ; jpe = ep32 ; kps = sp33 ; kpe = ep33 ; - imsx = sm31x ; imex = em31x ; jmsx = sm32x ; jmex = em32x ; kmsx = sm33x ; kmex = em33x ; - ipsx = sp31x ; ipex = ep31x ; jpsx = sp32x ; jpex = ep32x ; kpsx = sp33x ; kpex = ep33x ; - imsy = sm31y ; imey = em31y ; jmsy = sm32y ; jmey = em32y ; kmsy = sm33y ; kmey = em33y ; - ipsy = sp31y ; ipey = ep31y ; jpsy = sp32y ; jpey = ep32y ; kpsy = sp33y ; kpey = ep33y ; - CASE ( DATA_ORDER_YXZ ) - ids = sd32 ; ide = ed32 ; jds = sd31 ; jde = ed31 ; kds = sd33 ; kde = ed33 ; - ims = sm32 ; ime = em32 ; jms = sm31 ; jme = em31 ; kms = sm33 ; kme = em33 ; - ips = sp32 ; ipe = ep32 ; jps = sp31 ; jpe = ep31 ; kps = sp33 ; kpe = ep33 ; - imsx = sm32x ; imex = em32x ; jmsx = sm31x ; jmex = em31x ; kmsx = sm33x ; kmex = em33x ; - ipsx = sp32x ; ipex = ep32x ; jpsx = sp31x ; jpex = ep31x ; kpsx = sp33x ; kpex = ep33x ; - imsy = sm32y ; imey = em32y ; jmsy = sm31y ; jmey = em31y ; kmsy = sm33y ; kmey = em33y ; - ipsy = sp32y ; ipey = ep32y ; jpsy = sp31y ; jpey = ep31y ; kpsy = sp33y ; kpey = ep33y ; - CASE ( DATA_ORDER_ZXY ) - ids = sd32 ; ide = ed32 ; jds = sd33 ; jde = ed33 ; kds = sd31 ; kde = ed31 ; - ims = sm32 ; ime = em32 ; jms = sm33 ; jme = em33 ; kms = sm31 ; kme = em31 ; - ips = sp32 ; ipe = ep32 ; jps = sp33 ; jpe = ep33 ; kps = sp31 ; kpe = ep31 ; - imsx = sm32x ; imex = em32x ; jmsx = sm33x ; jmex = em33x ; kmsx = sm31x ; kmex = em31x ; - ipsx = sp32x ; ipex = ep32x ; jpsx = sp33x ; jpex = ep33x ; kpsx = sp31x ; kpex = ep31x ; - imsy = sm32y ; imey = em32y ; jmsy = sm33y ; jmey = em33y ; kmsy = sm31y ; kmey = em31y ; - ipsy = sp32y ; ipey = ep32y ; jpsy = sp33y ; jpey = ep33y ; kpsy = sp31y ; kpey = ep31y ; - CASE ( DATA_ORDER_ZYX ) - ids = sd33 ; ide = ed33 ; jds = sd32 ; jde = ed32 ; kds = sd31 ; kde = ed31 ; - ims = sm33 ; ime = em33 ; jms = sm32 ; jme = em32 ; kms = sm31 ; kme = em31 ; - ips = sp33 ; ipe = ep33 ; jps = sp32 ; jpe = ep32 ; kps = sp31 ; kpe = ep31 ; - imsx = sm33x ; imex = em33x ; jmsx = sm32x ; jmex = em32x ; kmsx = sm31x ; kmex = em31x ; - ipsx = sp33x ; ipex = ep33x ; jpsx = sp32x ; jpex = ep32x ; kpsx = sp31x ; kpex = ep31x ; - imsy = sm33y ; imey = em33y ; jmsy = sm32y ; jmey = em32y ; kmsy = sm31y ; kmey = em31y ; - ipsy = sp33y ; ipey = ep33y ; jpsy = sp32y ; jpey = ep32y ; kpsy = sp31y ; kpey = ep31y ; - CASE ( DATA_ORDER_XZY ) - ids = sd31 ; ide = ed31 ; jds = sd33 ; jde = ed33 ; kds = sd32 ; kde = ed32 ; - ims = sm31 ; ime = em31 ; jms = sm33 ; jme = em33 ; kms = sm32 ; kme = em32 ; - ips = sp31 ; ipe = ep31 ; jps = sp33 ; jpe = ep33 ; kps = sp32 ; kpe = ep32 ; - imsx = sm31x ; imex = em31x ; jmsx = sm33x ; jmex = em33x ; kmsx = sm32x ; kmex = em32x ; - ipsx = sp31x ; ipex = ep31x ; jpsx = sp33x ; jpex = ep33x ; kpsx = sp32x ; kpex = ep32x ; - imsy = sm31y ; imey = em31y ; jmsy = sm33y ; jmey = em33y ; kmsy = sm32y ; kmey = em32y ; - ipsy = sp31y ; ipey = ep31y ; jpsy = sp33y ; jpey = ep33y ; kpsy = sp32y ; kpey = ep32y ; - CASE ( DATA_ORDER_YZX ) - ids = sd33 ; ide = ed33 ; jds = sd31 ; jde = ed31 ; kds = sd32 ; kde = ed32 ; - ims = sm33 ; ime = em33 ; jms = sm31 ; jme = em31 ; kms = sm32 ; kme = em32 ; - ips = sp33 ; ipe = ep33 ; jps = sp31 ; jpe = ep31 ; kps = sp32 ; kpe = ep32 ; - imsx = sm33x ; imex = em33x ; jmsx = sm31x ; jmex = em31x ; kmsx = sm32x ; kmex = em32x ; - ipsx = sp33x ; ipex = ep33x ; jpsx = sp31x ; jpex = ep31x ; kpsx = sp32x ; kpex = ep32x ; - imsy = sm33y ; imey = em33y ; jmsy = sm31y ; jmey = em31y ; kmsy = sm32y ; kmey = em32y ; - ipsy = sp33y ; ipey = ep33y ; jpsy = sp31y ; jpey = ep31y ; kpsy = sp32y ; kpey = ep32y ; - END SELECT data_ordering - - CALL model_to_grid_config_rec ( id , model_config_rec , config_flags ) - - CALL nl_get_sr_x( id , sr_x ) - CALL nl_get_sr_y( id , sr_y ) - - tl = tl_in - inter_domain = inter_domain_in - okay_to_alloc = okay_to_alloc_in - -#if ( RWORDSIZE == 8 ) - initial_data_value = 0. -#else - CALL get_initial_data_value ( initial_data_value ) -#endif - -#ifdef NO_INITIAL_DATA_VALUE - setinitval = 0 -#else - setinitval = setinitval_in -#endif - - CALL nl_get_spec_bdy_width( 1, spec_bdy_width ) - -# include "allocs.inc" - - END SUBROUTINE ROUTINENAME diff --git a/frame/module_alloc_space_0.F b/frame/module_alloc_space_0.F deleted file mode 100644 index 5954c3331f..0000000000 --- a/frame/module_alloc_space_0.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 0 -#define ROUTINENAME alloc_space_field_core_0 -MODULE module_alloc_space_0 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_0 - diff --git a/frame/module_alloc_space_1.F b/frame/module_alloc_space_1.F deleted file mode 100644 index 5238bf95c9..0000000000 --- a/frame/module_alloc_space_1.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 1 -#define ROUTINENAME alloc_space_field_core_1 -MODULE module_alloc_space_1 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_1 - diff --git a/frame/module_alloc_space_2.F b/frame/module_alloc_space_2.F deleted file mode 100644 index 367b9b2699..0000000000 --- a/frame/module_alloc_space_2.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 2 -#define ROUTINENAME alloc_space_field_core_2 -MODULE module_alloc_space_2 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_2 - diff --git a/frame/module_alloc_space_3.F b/frame/module_alloc_space_3.F deleted file mode 100644 index 31d7213827..0000000000 --- a/frame/module_alloc_space_3.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 3 -#define ROUTINENAME alloc_space_field_core_3 -MODULE module_alloc_space_3 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_3 - diff --git a/frame/module_alloc_space_4.F b/frame/module_alloc_space_4.F deleted file mode 100644 index 3d7f0b51a9..0000000000 --- a/frame/module_alloc_space_4.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 4 -#define ROUTINENAME alloc_space_field_core_4 -MODULE module_alloc_space_4 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_4 - diff --git a/frame/module_alloc_space_5.F b/frame/module_alloc_space_5.F deleted file mode 100644 index db2974dc9d..0000000000 --- a/frame/module_alloc_space_5.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 5 -#define ROUTINENAME alloc_space_field_core_5 -MODULE module_alloc_space_5 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_5 - diff --git a/frame/module_alloc_space_6.F b/frame/module_alloc_space_6.F deleted file mode 100644 index 99423e3aa3..0000000000 --- a/frame/module_alloc_space_6.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 6 -#define ROUTINENAME alloc_space_field_core_6 -MODULE module_alloc_space_6 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_6 - diff --git a/frame/module_alloc_space_7.F b/frame/module_alloc_space_7.F deleted file mode 100644 index b172ea8074..0000000000 --- a/frame/module_alloc_space_7.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 7 -#define ROUTINENAME alloc_space_field_core_7 -MODULE module_alloc_space_7 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_7 - diff --git a/frame/module_alloc_space_8.F b/frame/module_alloc_space_8.F deleted file mode 100644 index 9d8da6742e..0000000000 --- a/frame/module_alloc_space_8.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 8 -#define ROUTINENAME alloc_space_field_core_8 -MODULE module_alloc_space_8 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_8 - diff --git a/frame/module_alloc_space_9.F b/frame/module_alloc_space_9.F deleted file mode 100644 index ea8abe81dc..0000000000 --- a/frame/module_alloc_space_9.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 9 -#define ROUTINENAME alloc_space_field_core_9 -MODULE module_alloc_space_9 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_9 - diff --git a/frame/module_domain.F b/frame/module_domain.F index 5633d3cca5..3a1d3c9bfb 100644 --- a/frame/module_domain.F +++ b/frame/module_domain.F @@ -326,7 +326,6 @@ SUBROUTINE get_ijk_from_subgrid ( grid , & END SUBROUTINE get_ijk_from_subgrid #endif - ! Default version ; Otherwise module containing interface to DM library will provide SUBROUTINE wrf_patch_domain( id , domdesc , parent, parent_id , parent_domdesc , & @@ -1295,19 +1294,11 @@ SUBROUTINE alloc_space_field ( grid, id, setinitval_in , tl_in , inter_domain sm31x, em31x, sm32x, em32x, sm33x, em33x, & sm31y, em31y, sm32y, em32y, sm33y, em33y ) - USE module_alloc_space_0, ONLY : alloc_space_field_core_0 - USE module_alloc_space_1, ONLY : alloc_space_field_core_1 - USE module_alloc_space_2, ONLY : alloc_space_field_core_2 - USE module_alloc_space_3, ONLY : alloc_space_field_core_3 - USE module_alloc_space_4, ONLY : alloc_space_field_core_4 - USE module_alloc_space_5, ONLY : alloc_space_field_core_5 - USE module_alloc_space_6, ONLY : alloc_space_field_core_6 - USE module_alloc_space_7, ONLY : alloc_space_field_core_7 - USE module_alloc_space_8, ONLY : alloc_space_field_core_8 - USE module_alloc_space_9, ONLY : alloc_space_field_core_9 IMPLICIT NONE +#include "allocs.inc" + ! Input data. TYPE(domain) , POINTER :: grid @@ -1344,87 +1335,8 @@ SUBROUTINE alloc_space_field ( grid, id, setinitval_in , tl_in , inter_domain num_bytes_allocated = 0 - ! now separate modules to reduce the size of module_domain that the compiler sees - CALL alloc_space_field_core_0 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_1 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_2 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_3 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_4 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_5 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_6 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_7 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_8 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_9 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) + ! Now use auto-split routines to allocate +#include "allocs_calls.inc" IF ( .NOT. grid%have_displayed_alloc_stats ) THEN ! we do not want to see this message more than once, as can happen with the allocation and diff --git a/main/depend.common b/main/depend.common index c1f5dc2526..f76684e3b9 100644 --- a/main/depend.common +++ b/main/depend.common @@ -81,16 +81,38 @@ module_dm_stubs.F: \ module_domain.o: \ module_domain_type.o \ - module_alloc_space_0.o \ - module_alloc_space_1.o \ - module_alloc_space_2.o \ - module_alloc_space_3.o \ - module_alloc_space_4.o \ - module_alloc_space_5.o \ - module_alloc_space_6.o \ - module_alloc_space_7.o \ - module_alloc_space_8.o \ - module_alloc_space_9.o \ + ../inc/allocs_0.o \ + ../inc/allocs_1.o \ + ../inc/allocs_2.o \ + ../inc/allocs_3.o \ + ../inc/allocs_4.o \ + ../inc/allocs_5.o \ + ../inc/allocs_6.o \ + ../inc/allocs_7.o \ + ../inc/allocs_8.o \ + ../inc/allocs_9.o \ + ../inc/allocs_10.o \ + ../inc/allocs_11.o \ + ../inc/allocs_12.o \ + ../inc/allocs_13.o \ + ../inc/allocs_14.o \ + ../inc/allocs_15.o \ + ../inc/allocs_16.o \ + ../inc/allocs_17.o \ + ../inc/allocs_18.o \ + ../inc/allocs_19.o \ + ../inc/allocs_20.o \ + ../inc/allocs_21.o \ + ../inc/allocs_22.o \ + ../inc/allocs_23.o \ + ../inc/allocs_24.o \ + ../inc/allocs_25.o \ + ../inc/allocs_26.o \ + ../inc/allocs_27.o \ + ../inc/allocs_28.o \ + ../inc/allocs_29.o \ + ../inc/allocs_30.o \ + ../inc/allocs_31.o \ module_driver_constants.o \ module_configure.o \ module_machine.o \ @@ -104,55 +126,233 @@ module_domain_type.o: \ module_streams.o \ $(ESMF_MOD_DEPENDENCE) +# For the following allocs_* source files, as they are generated by the registry +# we must make that a dependency. There is no actual registry target and is +# instead facilitated via the module_state_description.F recipe. Placing a dependency on +# module_state_description.F forces the registry to be run if the following source +# files are required implicitly by their respective .o recipe +../inc/allocs_0.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_0.F : module_state_description.F -module_alloc_space_0.o: \ + +../inc/allocs_1.o: \ module_domain_type.o \ - module_configure.o + module_configure.o +../inc/allocs_1.F : module_state_description.F -module_alloc_space_1.o: \ + +../inc/allocs_2.o: \ module_domain_type.o \ - module_configure.o + module_configure.o + +../inc/allocs_2.F : module_state_description.F -module_alloc_space_2.o: \ +../inc/allocs_3.o: \ module_domain_type.o \ - module_configure.o + module_configure.o + +../inc/allocs_3.F : module_state_description.F -module_alloc_space_3.o: \ +../inc/allocs_4.o: \ module_domain_type.o \ - module_configure.o + module_configure.o + +../inc/allocs_4.F : module_state_description.F -module_alloc_space_4.o: \ +../inc/allocs_5.o: \ module_domain_type.o \ - module_configure.o + module_configure.o + +../inc/allocs_5.F : module_state_description.F -module_alloc_space_5.o: \ +../inc/allocs_6.o: \ module_domain_type.o \ - module_configure.o + module_configure.o +../inc/allocs_6.F : module_state_description.F -module_alloc_space_6.o: \ + +../inc/allocs_7.o: \ module_domain_type.o \ - module_configure.o + module_configure.o +../inc/allocs_7.F : module_state_description.F -module_alloc_space_7.o: \ + +../inc/allocs_8.o: \ module_domain_type.o \ - module_configure.o + module_configure.o +../inc/allocs_8.F : module_state_description.F -module_alloc_space_8.o: \ + +../inc/allocs_9.o: \ module_domain_type.o \ - module_configure.o + module_configure.o + +../inc/allocs_9.F : module_state_description.F -module_alloc_space_9.o: \ +../inc/allocs_10.o: \ module_domain_type.o \ - module_configure.o + module_configure.o + +../inc/allocs_10.F : module_state_description.F + + +../inc/allocs_11.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_11.F : module_state_description.F + + +../inc/allocs_12.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_12.F : module_state_description.F + + +../inc/allocs_13.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_13.F : module_state_description.F + + +../inc/allocs_14.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_14.F : module_state_description.F + + +../inc/allocs_15.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_15.F : module_state_description.F + + +../inc/allocs_16.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_16.F : module_state_description.F + + +../inc/allocs_17.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_17.F : module_state_description.F + + +../inc/allocs_18.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_18.F : module_state_description.F + + +../inc/allocs_19.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_19.F : module_state_description.F + + +../inc/allocs_20.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_20.F : module_state_description.F + + +../inc/allocs_21.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_21.F : module_state_description.F + + +../inc/allocs_22.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_22.F : module_state_description.F + + +../inc/allocs_23.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_23.F : module_state_description.F + + +../inc/allocs_24.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_24.F : module_state_description.F + + +../inc/allocs_25.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_25.F : module_state_description.F + + +../inc/allocs_26.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_26.F : module_state_description.F + + +../inc/allocs_27.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_27.F : module_state_description.F + + +../inc/allocs_28.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_28.F : module_state_description.F + + +../inc/allocs_29.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_29.F : module_state_description.F + + +../inc/allocs_30.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_30.F : module_state_description.F + + +../inc/allocs_31.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_31.F : module_state_description.F module_streams.o: \ diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 1181ab0af4..1ac31ae7b3 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -99,6 +99,16 @@ elseif ( ${WRF_CORE} STREQUAL "DA" OR ${WRF_CORE} STREQUAL "DA_4D_VAR" ) endif() +set( allocs_source ) +foreach( n RANGE 0 31 ) + list( + APPEND + allocs_source + ${CMAKE_BINARY_DIR}/inc/allocs_${n}.F + ) + +endforeach() + get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) wrf_expand_definitions( RESULT_VAR REGISTRY_DEFS @@ -112,6 +122,9 @@ add_custom_command( OUTPUT ${CMAKE_BINARY_DIR}/inc/nl_config.inc ${CMAKE_BINARY_DIR}/frame/module_state_description.F + ${allocs_source} + COMMENT + "Generating registry code..." WORKING_DIRECTORY ${CMAKE_BINARY_DIR} # Replicate what exists in project directory for registry @@ -138,5 +151,12 @@ add_custom_target( DEPENDS ${CMAKE_BINARY_DIR}/inc/nl_config.inc ${CMAKE_BINARY_DIR}/frame/module_state_description.F + ${allocs_source} ) +target_sources( + ${PROJECT_NAME}_Core + PRIVATE + ${allocs_source} + ) + diff --git a/tools/gen_allocs.c b/tools/gen_allocs.c index abafbcb893..cff415550f 100644 --- a/tools/gen_allocs.c +++ b/tools/gen_allocs.c @@ -25,27 +25,252 @@ int gen_alloc1 ( char * dirname ) { FILE * fp ; + FILE * fpCalls ; char fname[NAMELEN] ; char * fn = "allocs.inc" ; - int startpiece, fraction, iguy, numguys ; - int stats[4] ; -#define FRAC 8 + char * fnCalls = "allocs_calls.inc" ; + node_t *p; + + // Open array of allocs_[n].F + int numFiles = 32; + int idx = 0; + int start = 0; + int stop = -1; + int primaryFields = 0; + FILE * fpSub; + char * filename_prefix = "allocs_" ; if ( dirname == NULL ) return(1) ; if ( strlen(dirname) > 0 ) { sprintf(fname,"%s/%s",dirname,fn) ; } else { sprintf(fname,"%s",fn) ; } if ((fp = fopen( fname , "w" )) == NULL ) return(1) ; + + if ( strlen(dirname) > 0 ) { sprintf(fname,"%s/%s",dirname,fnCalls) ; } + else { sprintf(fname,"%s",fnCalls) ; } + if ((fpCalls = fopen( fname , "w" )) == NULL ) return(1) ; + + for ( p = Domain.fields; p != NULL ; p = p->next ) { primaryFields++ ; } + print_warning(fp,fname) ; - startpiece = 0 ; - fraction = 0 ; - numguys = 0 ; - iguy = -1 ; - stats[0] = 0 ; stats[1] = 0 ; stats[2] = 0 ; stats[3] = 0 ; - get_count_for_alloc( &Domain, &numguys , stats) ; /* howmany deez guys? */ - fprintf(stderr,"Registry INFO variable counts: 0d %d 1d %d 2d %d 3d %d\n",stats[0],stats[1],stats[2],stats[3]) ; - fprintf(fp,"#if 1\n") ; - gen_alloc2( fp , "grid%", NULL, &Domain, &startpiece , &iguy, &fraction, numguys, FRAC, 1 ) ; - fprintf(fp,"#endif\n") ; + fprintf( + fp, + "INTERFACE\n" + ); + for ( idx = 0; idx < numFiles; idx++ ) + { + if ( strlen(dirname) > 0 ) { sprintf(fname,"%s/%s%d.F",dirname,filename_prefix,idx) ; } + else { sprintf(fname,"%s%d.F",dirname,filename_prefix,idx ) ; } + if ((fpSub = fopen( fname , "w" )) == NULL ) return(1) ; + + print_warning(fpSub,fname) ; + fprintf( + fp, + " SUBROUTINE %s%d( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , &\n" + " sd31, ed31, sd32, ed32, sd33, ed33, &\n" + " sm31 , em31 , sm32 , em32 , sm33 , em33 , &\n" + " sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , &\n" + " sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, &\n" + " sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, &\n" + " sm31x, em31x, sm32x, em32x, sm33x, em33x, &\n" + " sm31y, em31y, sm32y, em32y, sm33y, em33y )\n" + " USE module_domain_type\n" + " USE module_configure, ONLY : model_config_rec, grid_config_rec_type, in_use_for_config, model_to_grid_config_rec\n" + " USE module_scalar_tables ! this includes module_state_description too\n" + " IMPLICIT NONE\n" + " ! Input data.\n\n" + " TYPE(domain) , POINTER :: grid\n" + " INTEGER , INTENT(IN) :: id\n" + " INTEGER , INTENT(IN) :: setinitval_in ! 3 = everything, 1 = arrays only, 0 = none\n" + " INTEGER , INTENT(IN) :: sd31, ed31, sd32, ed32, sd33, ed33\n" + " INTEGER , INTENT(IN) :: sm31, em31, sm32, em32, sm33, em33\n" + " INTEGER , INTENT(IN) :: sp31, ep31, sp32, ep32, sp33, ep33\n" + " INTEGER , INTENT(IN) :: sp31x, ep31x, sp32x, ep32x, sp33x, ep33x\n" + " INTEGER , INTENT(IN) :: sp31y, ep31y, sp32y, ep32y, sp33y, ep33y\n" + " INTEGER , INTENT(IN) :: sm31x, em31x, sm32x, em32x, sm33x, em33x\n" + " INTEGER , INTENT(IN) :: sm31y, em31y, sm32y, em32y, sm33y, em33y\n\n" + " ! this argument is a bitmask. First bit is time level 1, second is time level 2, and so on.\n" + " ! e.g. to set both 1st and second time level, use 3\n" + " ! to set only 1st use 1\n" + " ! to set only 2st use 2\n" + " INTEGER , INTENT(IN) :: tl_in\n\n" + " ! true if the allocation is for an intermediate domain (for nesting); only certain fields allocated\n" + " ! false otherwise (all allocated, modulo tl above)\n" + " LOGICAL , INTENT(IN) :: inter_domain_in, okay_to_alloc_in\n\n" + " INTEGER(KIND=8) , INTENT(INOUT) :: num_bytes_allocated\n" + " END SUBROUTINE %s%d\n", + filename_prefix, idx, filename_prefix, idx + ); + + // Call the functions in the calls inc + fprintf( + fpCalls, + "CALL %s%d( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , &\n" + " sd31, ed31, sd32, ed32, sd33, ed33, &\n" + " sm31 , em31 , sm32 , em32 , sm33 , em33 , &\n" + " sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , &\n" + " sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, &\n" + " sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, &\n" + " sm31x, em31x, sm32x, em32x, sm33x, em33x, &\n" + " sm31y, em31y, sm32y, em32y, sm33y, em33y )\n", + filename_prefix, idx + ); + + fprintf( + fpSub, + "SUBROUTINE %s%d( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , &\n" + " sd31, ed31, sd32, ed32, sd33, ed33, &\n" + " sm31 , em31 , sm32 , em32 , sm33 , em33 , &\n" + " sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , &\n" + " sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, &\n" + " sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, &\n" + " sm31x, em31x, sm32x, em32x, sm33x, em33x, &\n" + " sm31y, em31y, sm32y, em32y, sm33y, em33y )\n" + " USE module_domain_type\n" + " USE module_configure, ONLY : model_config_rec, grid_config_rec_type, in_use_for_config, model_to_grid_config_rec\n" + " USE module_scalar_tables ! this includes module_state_description too\n" + " IMPLICIT NONE\n" + " ! Input data.\n\n" + " TYPE(domain) , POINTER :: grid\n" + " INTEGER , INTENT(IN) :: id\n" + " INTEGER , INTENT(IN) :: setinitval_in ! 3 = everything, 1 = arrays only, 0 = none\n" + " INTEGER , INTENT(IN) :: sd31, ed31, sd32, ed32, sd33, ed33\n" + " INTEGER , INTENT(IN) :: sm31, em31, sm32, em32, sm33, em33\n" + " INTEGER , INTENT(IN) :: sp31, ep31, sp32, ep32, sp33, ep33\n" + " INTEGER , INTENT(IN) :: sp31x, ep31x, sp32x, ep32x, sp33x, ep33x\n" + " INTEGER , INTENT(IN) :: sp31y, ep31y, sp32y, ep32y, sp33y, ep33y\n" + " INTEGER , INTENT(IN) :: sm31x, em31x, sm32x, em32x, sm33x, em33x\n" + " INTEGER , INTENT(IN) :: sm31y, em31y, sm32y, em32y, sm33y, em33y\n\n" + " ! this argument is a bitmask. First bit is time level 1, second is time level 2, and so on.\n" + " ! e.g. to set both 1st and second time level, use 3\n" + " ! to set only 1st use 1\n" + " ! to set only 2st use 2\n" + " INTEGER , INTENT(IN) :: tl_in\n\n" + " ! true if the allocation is for an intermediate domain (for nesting); only certain fields allocated\n" + " ! false otherwise (all allocated, modulo tl above)\n" + " LOGICAL , INTENT(IN) :: inter_domain_in, okay_to_alloc_in\n\n" + " INTEGER(KIND=8) , INTENT(INOUT) :: num_bytes_allocated\n" + " ! Local data.\n" + " INTEGER idum1, idum2, spec_bdy_width\n" + " REAL initial_data_value\n" + " CHARACTER (LEN=256) message\n" + " INTEGER tl\n" + " LOGICAL inter_domain, okay_to_alloc\n" + " INTEGER setinitval\n" + " INTEGER sr_x, sr_y\n\n" + " !declare ierr variable for error checking ALLOCATE calls\n" + " INTEGER ierr\n\n" + " INTEGER :: loop\n" + " INTEGER(KIND=8) :: nba ! number of bytes allocated per variable\n" + " CHARACTER(LEN=256) :: message_string\n\n" + " ! Local data\n\n" + " TYPE ( grid_config_rec_type ) :: config_flags\n\n" + " INTEGER :: k_start , k_end, its, ite, jts, jte\n" + " INTEGER :: ids , ide , jds , jde , kds , kde , &\n" + " ims , ime , jms , jme , kms , kme , &\n" + " ips , ipe , jps , jpe , kps , kpe\n\n" + " INTEGER :: sids , side , sjds , sjde , skds , skde , &\n" + " sims , sime , sjms , sjme , skms , skme , &\n" + " sips , sipe , sjps , sjpe , skps , skpe\n\n" + " INTEGER :: imsx, imex, jmsx, jmex, kmsx, kmex, &\n" + " ipsx, ipex, jpsx, jpex, kpsx, kpex, &\n" + " imsy, imey, jmsy, jmey, kmsy, kmey, &\n" + " ipsy, ipey, jpsy, jpey, kpsy, kpey\n\n" + " data_ordering : SELECT CASE ( model_data_order )\n" + " CASE ( DATA_ORDER_XYZ )\n" + " ids = sd31 ; ide = ed31 ; jds = sd32 ; jde = ed32 ; kds = sd33 ; kde = ed33 ;\n" + " ims = sm31 ; ime = em31 ; jms = sm32 ; jme = em32 ; kms = sm33 ; kme = em33 ;\n" + " ips = sp31 ; ipe = ep31 ; jps = sp32 ; jpe = ep32 ; kps = sp33 ; kpe = ep33 ;\n" + " imsx = sm31x ; imex = em31x ; jmsx = sm32x ; jmex = em32x ; kmsx = sm33x ; kmex = em33x ;\n" + " ipsx = sp31x ; ipex = ep31x ; jpsx = sp32x ; jpex = ep32x ; kpsx = sp33x ; kpex = ep33x ;\n" + " imsy = sm31y ; imey = em31y ; jmsy = sm32y ; jmey = em32y ; kmsy = sm33y ; kmey = em33y ;\n" + " ipsy = sp31y ; ipey = ep31y ; jpsy = sp32y ; jpey = ep32y ; kpsy = sp33y ; kpey = ep33y ;\n" + " CASE ( DATA_ORDER_YXZ )\n" + " ids = sd32 ; ide = ed32 ; jds = sd31 ; jde = ed31 ; kds = sd33 ; kde = ed33 ;\n" + " ims = sm32 ; ime = em32 ; jms = sm31 ; jme = em31 ; kms = sm33 ; kme = em33 ;\n" + " ips = sp32 ; ipe = ep32 ; jps = sp31 ; jpe = ep31 ; kps = sp33 ; kpe = ep33 ;\n" + " imsx = sm32x ; imex = em32x ; jmsx = sm31x ; jmex = em31x ; kmsx = sm33x ; kmex = em33x ;\n" + " ipsx = sp32x ; ipex = ep32x ; jpsx = sp31x ; jpex = ep31x ; kpsx = sp33x ; kpex = ep33x ;\n" + " imsy = sm32y ; imey = em32y ; jmsy = sm31y ; jmey = em31y ; kmsy = sm33y ; kmey = em33y ;\n" + " ipsy = sp32y ; ipey = ep32y ; jpsy = sp31y ; jpey = ep31y ; kpsy = sp33y ; kpey = ep33y ;\n" + " CASE ( DATA_ORDER_ZXY )\n" + " ids = sd32 ; ide = ed32 ; jds = sd33 ; jde = ed33 ; kds = sd31 ; kde = ed31 ;\n" + " ims = sm32 ; ime = em32 ; jms = sm33 ; jme = em33 ; kms = sm31 ; kme = em31 ;\n" + " ips = sp32 ; ipe = ep32 ; jps = sp33 ; jpe = ep33 ; kps = sp31 ; kpe = ep31 ;\n" + " imsx = sm32x ; imex = em32x ; jmsx = sm33x ; jmex = em33x ; kmsx = sm31x ; kmex = em31x ;\n" + " ipsx = sp32x ; ipex = ep32x ; jpsx = sp33x ; jpex = ep33x ; kpsx = sp31x ; kpex = ep31x ;\n" + " imsy = sm32y ; imey = em32y ; jmsy = sm33y ; jmey = em33y ; kmsy = sm31y ; kmey = em31y ;\n" + " ipsy = sp32y ; ipey = ep32y ; jpsy = sp33y ; jpey = ep33y ; kpsy = sp31y ; kpey = ep31y ;\n" + " CASE ( DATA_ORDER_ZYX )\n" + " ids = sd33 ; ide = ed33 ; jds = sd32 ; jde = ed32 ; kds = sd31 ; kde = ed31 ;\n" + " ims = sm33 ; ime = em33 ; jms = sm32 ; jme = em32 ; kms = sm31 ; kme = em31 ;\n" + " ips = sp33 ; ipe = ep33 ; jps = sp32 ; jpe = ep32 ; kps = sp31 ; kpe = ep31 ;\n" + " imsx = sm33x ; imex = em33x ; jmsx = sm32x ; jmex = em32x ; kmsx = sm31x ; kmex = em31x ;\n" + " ipsx = sp33x ; ipex = ep33x ; jpsx = sp32x ; jpex = ep32x ; kpsx = sp31x ; kpex = ep31x ;\n" + " imsy = sm33y ; imey = em33y ; jmsy = sm32y ; jmey = em32y ; kmsy = sm31y ; kmey = em31y ;\n" + " ipsy = sp33y ; ipey = ep33y ; jpsy = sp32y ; jpey = ep32y ; kpsy = sp31y ; kpey = ep31y ;\n" + " CASE ( DATA_ORDER_XZY )\n" + " ids = sd31 ; ide = ed31 ; jds = sd33 ; jde = ed33 ; kds = sd32 ; kde = ed32 ;\n" + " ims = sm31 ; ime = em31 ; jms = sm33 ; jme = em33 ; kms = sm32 ; kme = em32 ;\n" + " ips = sp31 ; ipe = ep31 ; jps = sp33 ; jpe = ep33 ; kps = sp32 ; kpe = ep32 ;\n" + " imsx = sm31x ; imex = em31x ; jmsx = sm33x ; jmex = em33x ; kmsx = sm32x ; kmex = em32x ;\n" + " ipsx = sp31x ; ipex = ep31x ; jpsx = sp33x ; jpex = ep33x ; kpsx = sp32x ; kpex = ep32x ;\n" + " imsy = sm31y ; imey = em31y ; jmsy = sm33y ; jmey = em33y ; kmsy = sm32y ; kmey = em32y ;\n" + " ipsy = sp31y ; ipey = ep31y ; jpsy = sp33y ; jpey = ep33y ; kpsy = sp32y ; kpey = ep32y ;\n" + " CASE ( DATA_ORDER_YZX )\n" + " ids = sd33 ; ide = ed33 ; jds = sd31 ; jde = ed31 ; kds = sd32 ; kde = ed32 ;\n" + " ims = sm33 ; ime = em33 ; jms = sm31 ; jme = em31 ; kms = sm32 ; kme = em32 ;\n" + " ips = sp33 ; ipe = ep33 ; jps = sp31 ; jpe = ep31 ; kps = sp32 ; kpe = ep32 ;\n" + " imsx = sm33x ; imex = em33x ; jmsx = sm31x ; jmex = em31x ; kmsx = sm32x ; kmex = em32x ;\n" + " ipsx = sp33x ; ipex = ep33x ; jpsx = sp31x ; jpex = ep31x ; kpsx = sp32x ; kpex = ep32x ;\n" + " imsy = sm33y ; imey = em33y ; jmsy = sm31y ; jmey = em31y ; kmsy = sm32y ; kmey = em32y ;\n" + " ipsy = sp33y ; ipey = ep33y ; jpsy = sp31y ; jpey = ep31y ; kpsy = sp32y ; kpey = ep32y ;\n" + " END SELECT data_ordering\n\n" + " CALL model_to_grid_config_rec ( id , model_config_rec , config_flags )\n\n" + " CALL nl_get_sr_x( id , sr_x )\n" + " CALL nl_get_sr_y( id , sr_y )\n\n" + " tl = tl_in\n" + " inter_domain = inter_domain_in\n" + " okay_to_alloc = okay_to_alloc_in\n\n" + "#if ( RWORDSIZE == 8 )\n" + " initial_data_value = 0.\n" + "#else\n" + " CALL get_initial_data_value ( initial_data_value )\n" + "#endif\n\n" + "#ifdef NO_INITIAL_DATA_VALUE\n" + " setinitval = 0\n" + "#else\n" + " setinitval = setinitval_in\n" + "#endif\n\n" + " CALL nl_get_spec_bdy_width( 1, spec_bdy_width )\n\n", + filename_prefix, idx + ); + + // Determine start/stop fields + start = stop + 1; + if ( idx == numFiles - 1 ) + { + // This should catch divisions that don't perfectly fit numFiles with at most numFiles + // extra fields in the last file + stop = primaryFields; + } + else + { + stop = start + ( primaryFields / numFiles ); + } + printf( "Total [%d] : Setting [start,stop] to [%d, %d]", primaryFields, start, stop ); + gen_alloc2( fpSub , "grid%", NULL, &Domain, start, stop, 1 ) ; + fprintf( + fpSub, + "END SUBROUTINE %s%d\n", + filename_prefix, idx + ); + } + fprintf( + fp, + "END INTERFACE\n" + ); + + close_the_file( fpCalls ) ; close_the_file( fp ) ; return(0) ; } @@ -77,7 +302,7 @@ int nolistthese( char * ) ; int -gen_alloc2 ( FILE * fp , char * structname , char * structname2 , node_t * node, int *j, int *iguy, int *fraction, int numguys, int frac, int sw ) /* 1 = allocate, 2 = just count */ +gen_alloc2 ( FILE * fp , char * structname , char * structname2 , node_t * node, int start, int stop, int sw ) /* 1 = allocate, 2 = just count */ { node_t * p ; int tag ; @@ -90,16 +315,23 @@ gen_alloc2 ( FILE * fp , char * structname , char * structname2 , node_t * node, unsigned int *io_mask ; int nd ; int restart ; + int currentIdx = -1; if ( node == NULL ) return(1) ; for ( p = node->fields ; p != NULL ; p = p->next ) { - (*iguy)++ ; - - if ( (*iguy % ((numguys+1)/frac+1)) == 0 ) { - fprintf(fp,"#endif\n") ; - fprintf(fp,"#if (NNN == %d)\n",(*j)++) ; + // Skip if this field is not part of [start,stop] and stop != -1, so -1 can be used to force output + currentIdx++;; + if ( currentIdx < start && stop != -1 ) + { + continue; + } + // We should be at [start] or forcing output via stop == -1 + if ( currentIdx > stop && stop != -1 ) + { + // We passed stop and are not forcing, exit loop + break; } nd = p->ndims + ((p->node_kind & FOURD)?1:0) ; @@ -489,7 +721,7 @@ if ( tag == 1 ) { sprintf(x,"%s%s%%",structname,p->name ) ; sprintf(x2,"%s%%",p->name ) ; - gen_alloc2(fp,x, x2, p->type, j, iguy, fraction, numguys, 1, sw) ; + gen_alloc2(fp,x, x2, p->type, start, -1, sw) ; } } } /* fraction loop */ diff --git a/tools/protos.h b/tools/protos.h index 3d39c2dfa7..6e9fa2dd98 100644 --- a/tools/protos.h +++ b/tools/protos.h @@ -70,7 +70,7 @@ char * get_typename_i(int i) ; int gen_alloc ( char * dirname ) ; int gen_alloc1 ( char * dirname ) ; -int gen_alloc2 ( FILE * fp , char * structname , char * structname2 , node_t * node, int *j, int *iguy, int *fraction, int numguys, int frac, int sw ); +int gen_alloc2 ( FILE * fp , char * structname , char * structname2 , node_t * node, int start, int stop, int sw ); int gen_module_state_description ( char * dirname ) ; int gen_module_state_description1 ( FILE * fp , node_t * node ) ; diff --git a/var/build/da.make b/var/build/da.make index 6770f48734..8ed639627c 100644 --- a/var/build/da.make +++ b/var/build/da.make @@ -125,16 +125,38 @@ WRFVAR_OBJS = \ module_comm_dm_2.o \ module_comm_dm_3.o \ module_comm_dm_4.o \ - module_alloc_space_0.o \ - module_alloc_space_1.o \ - module_alloc_space_2.o \ - module_alloc_space_3.o \ - module_alloc_space_4.o \ - module_alloc_space_5.o \ - module_alloc_space_6.o \ - module_alloc_space_7.o \ - module_alloc_space_8.o \ - module_alloc_space_9.o \ + inc/allocs_0.o \ + inc/allocs_1.o \ + inc/allocs_2.o \ + inc/allocs_3.o \ + inc/allocs_4.o \ + inc/allocs_5.o \ + inc/allocs_6.o \ + inc/allocs_7.o \ + inc/allocs_8.o \ + inc/allocs_9.o \ + inc/allocs_10.o \ + inc/allocs_11.o \ + inc/allocs_12.o \ + inc/allocs_13.o \ + inc/allocs_14.o \ + inc/allocs_15.o \ + inc/allocs_16.o \ + inc/allocs_17.o \ + inc/allocs_18.o \ + inc/allocs_19.o \ + inc/allocs_20.o \ + inc/allocs_21.o \ + inc/allocs_22.o \ + inc/allocs_23.o \ + inc/allocs_24.o \ + inc/allocs_25.o \ + inc/allocs_26.o \ + inc/allocs_27.o \ + inc/allocs_28.o \ + inc/allocs_29.o \ + inc/allocs_30.o \ + inc/allocs_31.o \ nl_get_0_routines.o \ nl_get_1_routines.o \ nl_get_2_routines.o \ diff --git a/var/build/da_name_space.pl b/var/build/da_name_space.pl index 170fe0315a..ba01fcd738 100755 --- a/var/build/da_name_space.pl +++ b/var/build/da_name_space.pl @@ -53,7 +53,7 @@ med_auxinput_in med_restart_out med_latbound_in -module_alloc_space +allocs_ module_configure module_scalar_tables module_nesting diff --git a/var/build/depend.txt b/var/build/depend.txt index 3d12fee59c..604d8ed045 100644 --- a/var/build/depend.txt +++ b/var/build/depend.txt @@ -12,16 +12,6 @@ mediation_interp_domain.o : mediation_interp_domain.F module_timing.o module_con mediation_nest_move.o : mediation_nest_move.F module_streams.o module_compute_geop.o module_driver_constants.o module_dm.o module_state_description.o module_configure.o module_timing.o module_utility.o module_domain.o mediation_wrfmain.o : mediation_wrfmain.F module_utility.o module_bc_time_utilities.o module_io.o module_timing.o module_io_domain.o module_configure.o module_domain.o module_MPP.o : module_MPP.F -module_alloc_space_0.o : module_alloc_space_0.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_1.o : module_alloc_space_1.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_2.o : module_alloc_space_2.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_3.o : module_alloc_space_3.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_4.o : module_alloc_space_4.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_5.o : module_alloc_space_5.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_6.o : module_alloc_space_6.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_7.o : module_alloc_space_7.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_8.o : module_alloc_space_8.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_9.o : module_alloc_space_9.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o module_bc.o : module_bc.F module_state_description.o module_wrf_error.o module_configure.o module_model_constants.o module_bc_time_utilities.o : module_bc_time_utilities.F module_utility.o module_check_a_mundo.o : module_check_a_mundo.F module_configure.o module_wrf_error.o module_state_description.o @@ -36,8 +26,171 @@ module_compute_geop.o : module_compute_geop.F module_configure.o : module_configure.F module_wrf_error.o module_domain_type.o module_state_description.o module_driver_constants.o module_date_time.o : module_date_time.F module_utility.o module_model_constants.o module_configure.o module_wrf_error.o module_dm.o : module_dm.F module_comm_dm.o module_utility.o module_comm_nesting_dm.o module_configure.o module_state_description.o module_timing.o module_domain.o module_driver_constants.o module_wrf_error.o module_machine.o -module_domain.o : module_domain.F module_alloc_space_9.o module_alloc_space_8.o module_alloc_space_7.o module_alloc_space_6.o module_alloc_space_5.o module_alloc_space_4.o module_alloc_space_3.o module_alloc_space_2.o module_alloc_space_1.o module_alloc_space_0.o module_domain_type.o module_utility.o module_wrf_error.o module_configure.o module_machine.o module_driver_constants.o +module_domain.o : module_domain.F \ + inc/allocs_0.o \ + inc/allocs_1.o \ + inc/allocs_2.o \ + inc/allocs_3.o \ + inc/allocs_4.o \ + inc/allocs_5.o \ + inc/allocs_6.o \ + inc/allocs_7.o \ + inc/allocs_8.o \ + inc/allocs_9.o \ + inc/allocs_10.o \ + inc/allocs_11.o \ + inc/allocs_12.o \ + inc/allocs_13.o \ + inc/allocs_14.o \ + inc/allocs_15.o \ + inc/allocs_16.o \ + inc/allocs_17.o \ + inc/allocs_18.o \ + inc/allocs_19.o \ + inc/allocs_20.o \ + inc/allocs_21.o \ + inc/allocs_22.o \ + inc/allocs_23.o \ + inc/allocs_24.o \ + inc/allocs_25.o \ + inc/allocs_26.o \ + inc/allocs_27.o \ + inc/allocs_28.o \ + inc/allocs_29.o \ + inc/allocs_30.o \ + inc/allocs_31.o \ + module_domain_type.o module_utility.o module_wrf_error.o module_configure.o module_machine.o module_driver_constants.o module_domain_type.o : module_domain_type.F module_streams.o module_utility.o module_driver_constants.o +# See main/depend.common for explanation of source file dependency recipes for allocs +inc/allocs_0.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_0.F : module_state_description.F +inc/allocs_1.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_1.F : module_state_description.F +inc/allocs_2.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_2.F : module_state_description.F +inc/allocs_3.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_3.F : module_state_description.F +inc/allocs_4.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_4.F : module_state_description.F +inc/allocs_5.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_5.F : module_state_description.F +inc/allocs_6.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_6.F : module_state_description.F +inc/allocs_7.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_7.F : module_state_description.F +inc/allocs_8.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_8.F : module_state_description.F +inc/allocs_9.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_9.F : module_state_description.F +inc/allocs_10.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_10.F : module_state_description.F +inc/allocs_11.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_11.F : module_state_description.F +inc/allocs_12.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_12.F : module_state_description.F +inc/allocs_13.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_13.F : module_state_description.F +inc/allocs_14.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_14.F : module_state_description.F +inc/allocs_15.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_15.F : module_state_description.F +inc/allocs_16.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_16.F : module_state_description.F +inc/allocs_17.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_17.F : module_state_description.F +inc/allocs_18.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_18.F : module_state_description.F +inc/allocs_19.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_19.F : module_state_description.F +inc/allocs_20.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_20.F : module_state_description.F +inc/allocs_21.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_21.F : module_state_description.F +inc/allocs_22.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_22.F : module_state_description.F +inc/allocs_23.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_23.F : module_state_description.F +inc/allocs_24.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_24.F : module_state_description.F +inc/allocs_25.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_25.F : module_state_description.F +inc/allocs_26.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_26.F : module_state_description.F +inc/allocs_27.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_27.F : module_state_description.F +inc/allocs_28.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_28.F : module_state_description.F +inc/allocs_29.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_29.F : module_state_description.F +inc/allocs_30.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_30.F : module_state_description.F +inc/allocs_31.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_31.F : module_state_description.F + module_driver_constants.o : module_driver_constants.F module_em.o : module_em.F module_state_description.o module_model_constants.o module_get_file_names.o : module_get_file_names.F From d810865d1f633b85e1077438f420e84170417bfd Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:21:17 -0700 Subject: [PATCH 16/30] Break out module_dm external subroutines into separate files (#2069) TYPE: enhancement KEYWORDS: intel, compilation, llvm, memory SOURCE: internal DESCRIPTION OF CHANGES: Problem: The Intel oneAPI compilers (and others like nvhpc) struggle with some of the larger (15k+ lines of code) files within WRF. This causes intense memory usage that is not often available to the average user not in a resource-rich environment. This often limits compilation to single threaded if even possible or to a dedicated environment with enough memory if available. If neither of those is available to a user, they will be unable to use these configurations entirely. Solution: This PR focuses on the `module_dm` sections of code to reduce its individual file size to manageable levels. This and its helper subroutines are instead broken out into many smaller files. TESTS CONDUCTED: Attached to this PR are plots of the respective effects of theses changes. Changes were tested with intel and gcc compilers, but only intel memory usage is shown as it exacerbates the memory usage issue. --- dyn_em/interp_domain_em.F | 3 + external/RSL_LITE/feedback_domain_em_part1.F | 111 ++ external/RSL_LITE/feedback_domain_em_part2.F | 200 +++ external/RSL_LITE/force_domain_em_part2.F | 314 +++++ external/RSL_LITE/interp_domain_em_part1.F | 105 ++ external/RSL_LITE/interp_domain_em_part2.F | 187 +++ external/RSL_LITE/interp_domain_em_part3.F | 59 + external/RSL_LITE/interp_domain_em_small.F | 411 ++++++ external/RSL_LITE/makefile | 15 +- external/RSL_LITE/module_dm.F | 1314 ------------------ frame/CMakeLists.txt | 12 +- frame/Makefile | 18 +- main/depend.common | 58 + share/mediation_interp_domain.F | 20 + 14 files changed, 1507 insertions(+), 1320 deletions(-) create mode 100644 external/RSL_LITE/feedback_domain_em_part1.F create mode 100644 external/RSL_LITE/feedback_domain_em_part2.F create mode 100644 external/RSL_LITE/force_domain_em_part2.F create mode 100644 external/RSL_LITE/interp_domain_em_part1.F create mode 100644 external/RSL_LITE/interp_domain_em_part2.F create mode 100644 external/RSL_LITE/interp_domain_em_part3.F create mode 100644 external/RSL_LITE/interp_domain_em_small.F diff --git a/dyn_em/interp_domain_em.F b/dyn_em/interp_domain_em.F index 1c97931a6e..739d73c400 100644 --- a/dyn_em/interp_domain_em.F +++ b/dyn_em/interp_domain_em.F @@ -55,6 +55,9 @@ END SUBROUTINE interp_domain_em_part1 SUBROUTINE interp_domain_em_part2 END SUBROUTINE interp_domain_em_part2 +SUBROUTINE interp_domain_em_part3 +END SUBROUTINE interp_domain_em_part3 + #endif diff --git a/external/RSL_LITE/feedback_domain_em_part1.F b/external/RSL_LITE/feedback_domain_em_part1.F new file mode 100644 index 0000000000..8fe36fc147 --- /dev/null +++ b/external/RSL_LITE/feedback_domain_em_part1.F @@ -0,0 +1,111 @@ +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + SUBROUTINE feedback_domain_em_part1 ( grid, ngrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type, model_config_rec, model_to_grid_config_rec + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, mytask, & + ipe_save, jpe_save, ips_save, jps_save, & + nest_pes_x, nest_pes_y + + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: ngrid +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + TYPE(domain), POINTER :: xgrid + TYPE (grid_config_rec_type) :: config_flags, nconfig_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER local_comm, myproc, nproc, idum1, idum2 + INTEGER thisdomain_max_halo_width + +!cyl: add variables for trajectory + integer tjk + + INTERFACE + SUBROUTINE feedback_nest_prep ( grid, config_flags & +! +#include "dummy_new_args.inc" +! +) + USE module_state_description + USE module_domain, ONLY : domain + USE module_configure, ONLY : grid_config_rec_type +! + TYPE (grid_config_rec_type) :: config_flags + TYPE(domain), TARGET :: grid +#include "dummy_new_decl.inc" + END SUBROUTINE feedback_nest_prep + END INTERFACE +! + + CALL wrf_get_dm_communicator ( local_comm ) + CALL wrf_get_myproc( myproc ) + CALL wrf_get_nproc( nproc ) + +! +! intermediate grid + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) +! nest grid + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + + nlev = ckde - ckds + 1 + + ips_save = ngrid%i_parent_start ! used in feedback_domain_em_part2 below + jps_save = ngrid%j_parent_start + ipe_save = ngrid%i_parent_start + (nide-nids+1) / ngrid%parent_grid_ratio - 1 + jpe_save = ngrid%j_parent_start + (njde-njds+1) / ngrid%parent_grid_ratio - 1 + +! feedback_nest_prep invokes a halo exchange on the ngrid. It is done this way +! in a separate routine because the HALOs need the data to be dereference from the +! grid data structure and, in this routine, the dereferenced fields are related to +! the intermediate domain, not the nest itself. Save the current grid pointer to intermediate +! domain, switch grid to point to ngrid, invoke feedback_nest_prep, then restore grid +! to point to intermediate domain. + + CALL model_to_grid_config_rec ( ngrid%id , model_config_rec , nconfig_flags ) + CALL set_scalar_indices_from_config ( ngrid%id , idum1 , idum2 ) + xgrid => grid + grid => ngrid + + CALL feedback_nest_prep ( grid, nconfig_flags & +! +#include "actual_new_args.inc" +! +) + +! put things back so grid is intermediate grid + + grid => xgrid + CALL set_scalar_indices_from_config ( grid%id , idum1 , idum2 ) + +! "interp" (basically copy) ngrid onto intermediate grid + +#include "nest_feedbackup_interp.inc" + + RETURN + END SUBROUTINE feedback_domain_em_part1 +#endif + diff --git a/external/RSL_LITE/feedback_domain_em_part2.F b/external/RSL_LITE/feedback_domain_em_part2.F new file mode 100644 index 0000000000..3096e9cef5 --- /dev/null +++ b/external/RSL_LITE/feedback_domain_em_part2.F @@ -0,0 +1,200 @@ +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + + SUBROUTINE feedback_domain_em_part2 ( grid, intermediate_grid, ngrid , config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, domain_clock_get, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type, model_config_rec + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, mytask, & + ipe_save, jpe_save, ips_save, jps_save, get_dm_max_halo_width, & + nest_pes_x, nest_pes_y, & + intercomm_active, nest_task_offsets, & + mpi_comm_to_mom, mpi_comm_to_kid, which_kid !, & + !push_communicators_for_domain, pop_communicators_for_domain + + USE module_comm_nesting_dm, ONLY : halo_interp_up_sub + USE module_utility + IMPLICIT NONE + +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: intermediate_grid + TYPE(domain), POINTER :: ngrid + TYPE(domain), POINTER :: parent_grid + +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + INTEGER :: xids, xide, xjds, xjde, xkds, xkde, & + xims, xime, xjms, xjme, xkms, xkme, & + xips, xipe, xjps, xjpe, xkps, xkpe + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER icoord, jcoord, idim_cd, jdim_cd + INTEGER local_comm, myproc, nproc, ioffset + INTEGER iparstrt, jparstrt, sw, thisdomain_max_halo_width + REAL nest_influence + + character*256 :: timestr + integer ierr + + LOGICAL, EXTERNAL :: cd_feedback_mask + +!cyl: add variables for trajectory + integer tjk + +! On entry to this routine, +! "grid" refers to the parent domain +! "intermediate_grid" refers to local copy of parent domain that overlies this patch of nest +! "ngrid" refers to the nest, which is only needed for smoothing on the parent because +! the nest feedback data has already been transferred during em_nest_feedbackup_interp +! in part1, above. +! The way these settings c and n dimensions are set, below, looks backwards but from the point +! of view of the RSL routine rsl_lite_to_parent_info(), call to which is included by +! em_nest_feedbackup_pack, the "n" domain represents the parent domain and the "c" domain +! represents the intermediate domain. The backwards lookingness should be fixed in the gen_comms.c +! registry routine that accompanies RSL_LITE but, just as it's sometimes easier to put up a road +! sign that says "DIP" than fix the dip, at this point it was easier just to write this comment. JM +! + nest_influence = 1. + + CALL domain_clock_get( grid, current_timestr=timestr ) + + CALL get_ijk_from_grid ( intermediate_grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + CALL get_ijk_from_grid ( grid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + CALL get_ijk_from_grid ( ngrid , & + xids, xide, xjds, xjde, xkds, xkde, & + xims, xime, xjms, xjme, xkms, xkme, & + xips, xipe, xjps, xjpe, xkps, xkpe ) + + ips_save = ngrid%i_parent_start ! used in feedback_domain_em_part2 below + jps_save = ngrid%j_parent_start + ipe_save = ngrid%i_parent_start + (xide-xids+1) / ngrid%parent_grid_ratio - 1 + jpe_save = ngrid%j_parent_start + (xjde-xjds+1) / ngrid%parent_grid_ratio - 1 + + + + +IF ( ngrid%active_this_task ) THEN +!cyl add this for trajectory + CALL push_communicators_for_domain( ngrid%id ) + + do tjk = 1,config_flags%num_traj + if (ngrid%traj_long(tjk) .eq. -9999.0) then +! print*,'n=-9999',tjk + ngrid%traj_long(tjk)=grid%traj_long(tjk) + ngrid%traj_k(tjk)=grid%traj_k(tjk) + else +! print*,'p!=-9999',tjk + grid%traj_long(tjk)=ngrid%traj_long(tjk) + grid%traj_k(tjk)=ngrid%traj_k(tjk) + endif + if (ngrid%traj_lat(tjk) .eq. -9999.0) then + ngrid%traj_lat(tjk)=grid%traj_lat(tjk) + ngrid%traj_k(tjk)=grid%traj_k(tjk) + else + grid%traj_lat(tjk)=ngrid%traj_lat(tjk) + grid%traj_k(tjk)=ngrid%traj_k(tjk) + endif + enddo +!endcyl + + CALL nl_get_i_parent_start ( intermediate_grid%id, iparstrt ) + CALL nl_get_j_parent_start ( intermediate_grid%id, jparstrt ) + CALL nl_get_shw ( intermediate_grid%id, sw ) + icoord = iparstrt - sw + jcoord = jparstrt - sw + idim_cd = cide - cids + 1 + jdim_cd = cjde - cjds + 1 + + nlev = ckde - ckds + 1 + + CALL get_dm_max_halo_width ( grid%id , thisdomain_max_halo_width ) + + parent_grid => grid + grid => ngrid +#include "nest_feedbackup_pack.inc" + grid => parent_grid + CALL pop_communicators_for_domain + +END IF + +! CALL wrf_get_dm_communicator ( local_comm ) +! CALL wrf_get_myproc( myproc ) +! CALL wrf_get_nproc( nproc ) + + ! determine which communicator and offset to use + IF ( intercomm_active( grid%id ) ) THEN ! I am parent + local_comm = mpi_comm_to_kid( which_kid(ngrid%id), grid%id ) + ioffset = nest_task_offsets(ngrid%id) + ELSE IF ( intercomm_active( ngrid%id ) ) THEN ! I am nest + local_comm = mpi_comm_to_mom( ngrid%id ) + ioffset = nest_task_offsets(ngrid%id) + END IF + + IF ( grid%active_this_task .OR. ngrid%active_this_task ) THEN +#ifndef STUBMPI + CALL mpi_comm_rank(local_comm,myproc,ierr) + CALL mpi_comm_size(local_comm,nproc,ierr) +#endif +!call tracebackqq() + CALL rsl_lite_merge_msgs( myproc, nest_pes_x(grid%id)*nest_pes_y(grid%id), & + nest_pes_x(ngrid%id)*nest_pes_y(ngrid%id), & + ioffset, local_comm ) + END IF + +IF ( grid%active_this_task ) THEN + CALL push_communicators_for_domain( grid%id ) + + +#define NEST_INFLUENCE(A,B) A = B +#include "nest_feedbackup_unpack.inc" + + ! smooth coarse grid + CALL get_ijk_from_grid ( ngrid, & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + CALL get_ijk_from_grid ( grid , & + ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe ) + +#include "HALO_INTERP_UP.inc" + + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + +#include "nest_feedbackup_smooth.inc" + + CALL pop_communicators_for_domain +END IF + + RETURN + END SUBROUTINE feedback_domain_em_part2 +#endif diff --git a/external/RSL_LITE/force_domain_em_part2.F b/external/RSL_LITE/force_domain_em_part2.F new file mode 100644 index 0000000000..8c6c4e7e98 --- /dev/null +++ b/external/RSL_LITE/force_domain_em_part2.F @@ -0,0 +1,314 @@ +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + + SUBROUTINE force_domain_em_part2 ( grid, ngrid, pgrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, local_communicator, mytask, & + nest_pes_x, nest_pes_y ! , & + !push_communicators_for_domain,pop_communicators_for_domain + USE module_comm_nesting_dm, ONLY : halo_force_down_sub + USE module_model_constants + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: ngrid + TYPE(domain), POINTER :: pgrid !KAL added for vertical nesting +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k,kk + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7,itrace + REAL dummy_xs, dummy_xe, dummy_ys, dummy_ye + + !KAL variables for vertical nesting + REAL :: p_top_m , p_surf_m , mu_m , hsca_m , pre_c ,pre_n + REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert) :: alt_w_c + REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert+1) :: alt_u_c + REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert) :: alt_w_n + REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert+1) :: alt_u_n + + REAL, DIMENSION(:,:,:), ALLOCATABLE :: p, al + REAL :: pfu, pfd, phm, temp, qvf, qvf1, qvf2 + + !KAL change this for vertical nesting + ! force_domain_em_part1 packs up the interpolation onto the coarse (vertical) grid + ! therefore the message size is based on the coarse grid number of levels + ! here it is unpacked onto the intermediate grid + CALL get_ijk_from_grid ( pgrid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + + !KAL this is the original WRF code + !CALL get_ijk_from_grid ( grid , & + ! cids, cide, cjds, cjde, ckds, ckde, & + ! cims, cime, cjms, cjme, ckms, ckme, & + ! cips, cipe, cjps, cjpe, ckps, ckpe ) + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + + nlev = ckde - ckds + 1 + +#include "nest_interpdown_unpack.inc" + +if (ngrid%vert_refine_method .NE. 0) then + + !KAL calculating the vertical coordinate for parent and nest grid (code from ndown) + ! assume that the parent and nest have the same p_top value (as in ndown) + +!KAL ckde is equal to e_vert of the coarse grid. There are e_vert-1 u points. The coarse 1D grid here is e_vert+1, +! so it is the e_vert-1 points from the coarse grid, plus a surface point plus a top point. Extrapolation coefficients +! are used to get the surface and top points to fill out the pro_u_c 1D array of u values from the coarse grid. + + hsca_m = 6.7 !KAL scale height of the atmosphere + p_top_m = ngrid%p_top + p_surf_m = 1.e5 + mu_m = p_surf_m - p_top_m +! parent + do k = 1,ckde + pre_c = mu_m * pgrid%c3f(k) + p_top_m + pgrid%c4f(k) + alt_w_c(k) = -hsca_m * alog(pre_c/p_surf_m) + enddo + do k = 1,ckde-1 + pre_c = mu_m * pgrid%c3h(k) + p_top_m + pgrid%c4h(k) + alt_u_c(k+1) = -hsca_m * alog(pre_c/p_surf_m) + enddo + alt_u_c(1) = alt_w_c(1) + alt_u_c(ckde+1) = alt_w_c(ckde) +! nest + do k = 1,nkde + pre_n = mu_m * ngrid%c3f(k) + p_top_m + ngrid%c4f(k) + alt_w_n(k) = -hsca_m * alog(pre_n/p_surf_m) + enddo + do k = 1,nkde-1 + pre_n = mu_m * ngrid%c3h(k) + p_top_m + ngrid%c4h(k) + alt_u_n(k+1) = -hsca_m * alog(pre_n/p_surf_m) + enddo + alt_u_n(1) = alt_w_n(1) + alt_u_n(nkde+1) = alt_w_n(nkde) + +endif + + !KAL added this call for vertical nesting (return coarse grid dimensions to intended values) + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + + CALL get_ijk_from_grid ( grid , & + ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe ) + + ! Vertical refinement is turned on. + + IF (ngrid%vert_refine_method .NE. 0) THEN + +#include "nest_forcedown_interp_vert.inc" + + IF ( ngrid%this_is_an_ideal_run ) THEN + IF ( SIZE( grid%t_init, 1 ) * SIZE( grid%t_init, 3 ) .GT. 1 ) THEN + CALL vert_interp_vert_nesting( grid%t_init, & !CD field + ids, ide, kds, kde, jds, jde, & !CD dims + ims, ime, kms, kme, jms, jme, & !CD dims + ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & !CD dims + pgrid%s_vert, pgrid%e_vert, & !vertical dimension of the parent grid + pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & !coarse grid extrapolation constants + alt_u_c, alt_u_n ) !coordinates for parent and nest + END IF ! Check t_init is a fully allocated 3d array. + END IF ! only for ideal runs + + + ! Rebalance the grid on the intermediate grid. The intermediate grid has the horizontal + ! resolution of the parent grid, but at this point has been interpolated in the vertical + ! to the resolution of the nest. The base state (phb, pb, etc) from the parent grid is + ! unpacked onto the intermediate grid every time this subroutine is called. We need the + ! base state of the nest, so it is recalculated here. + + ! Additionally, we do not need to vertically interpolate the entire intermediate grid + ! above, just the points that contribute to the boundary forcing. + + ! Base state potential temperature and inverse density (alpha = 1/rho) from + ! the half eta levels and the base-profile surface pressure. Compute 1/rho + ! from equation of state. The potential temperature is a perturbation from t0. + + ! Uncouple the variables moist and t_2 that are used to calculate ph_2 + + DO j = MAX(jds,jps),MIN(jde-1,jpe) + DO i = MAX(ids,ips),MIN(ide-1,ipe) + DO k=kds,kde-1 + grid%t_2(i,k,j) = grid%t_2(i,k,j)/((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) + moist(i,k,j,P_QV) = moist(i,k,j,P_QV)/((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) + END DO + END DO + END DO + + DO j = MAX(jds,jps),MIN(jde-1,jpe) + DO i = MAX(ids,ips),MIN(ide-1,ipe) + + DO k = 1, kpe-1 + grid%pb(i,k,j) = ngrid%c3h(k) * grid%mub(i,j) + ngrid%c4h(k) + ngrid%p_top + + ! If this is a real run, recalc t_init. + + IF ( .NOT. ngrid%this_is_an_ideal_run ) THEN + temp = MAX ( ngrid%tiso, ngrid%t00 + ngrid%tlp*LOG(grid%pb(i,k,j)/ngrid%p00) ) + IF ( grid%pb(i,k,j) .LT. ngrid%p_strat ) THEN + temp = ngrid%tiso + ngrid%tlp_strat * LOG ( grid%pb(i,k,j)/ngrid%p_strat ) + END IF + grid%t_init(i,k,j) = temp*(ngrid%p00/grid%pb(i,k,j))**(r_d/cp) - t0 + END IF + grid%alb(i,k,j) = (r_d/p1000mb)*(grid%t_init(i,k,j)+t0)*(grid%pb(i,k,j)/p1000mb)**cvpm + END DO + + ! Integrate base geopotential, starting at terrain elevation. This assures that + ! the base state is in exact hydrostatic balance with respect to the model equations. + ! This field is on full levels. + + grid%phb(i,1,j) = grid%ht(i,j) * g + IF (grid%hypsometric_opt == 1) THEN + DO kk = 2,kpe + k = kk - 1 + grid%phb(i,kk,j) = grid%phb(i,k,j) - ngrid%dnw(k)*(ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k))*grid%alb(i,k,j) + END DO + ELSE IF (grid%hypsometric_opt == 2) THEN + DO k = 2,kpe + pfu = ngrid%c3f(k )*grid%MUB(i,j) + ngrid%c4f(k ) + ngrid%p_top + pfd = ngrid%c3f(k-1)*grid%MUB(i,j) + ngrid%c4f(k-1) + ngrid%p_top + phm = ngrid%c3h(k-1)*grid%MUB(i,j) + ngrid%c4h(k-1) + ngrid%p_top + grid%phb(i,k,j) = grid%phb(i,k-1,j) + grid%alb(i,k-1,j)*phm*LOG(pfd/pfu) + END DO + ELSE + CALL wrf_error_fatal( 'module_dm: hypsometric_opt should be 1 or 2' ) + END IF ! which hypsometric option + END DO ! i loop + END DO ! j loop + ! Perturbation fields + ALLOCATE( p (ips:ipe, kps:kpe, jps:jpe) ) + ALLOCATE( al(ips:ipe, kps:kpe, jps:jpe) ) + DO j = MAX(jds,jps),MIN(jde-1,jpe) + DO i = MAX(ids,ips),MIN(ide-1,ipe) + ! Integrate the hydrostatic equation (from the RHS of the bigstep vertical momentum + ! equation) down from the top to get the pressure perturbation. First get the pressure + ! perturbation, moisture, and inverse density (total and perturbation) at the top-most level. + + kk = kpe-1 + k = kk+1 + + qvf1 = 0.5*(moist(i,kk,j,P_QV)+moist(i,kk,j,P_QV)) + qvf2 = 1./(1.+qvf1) + qvf1 = qvf1*qvf2 + + p(i,kk,j) = - 0.5*((ngrid%c1f(k)*grid%Mu_2(i,j))+qvf1*(ngrid%c1f(k)*grid%Mub(i,j)+ngrid%c2f(k)))/ngrid%rdnw(kk)/qvf2 + IF ( config_flags%use_theta_m == 0) THEN + qvf = 1. + rvovrd*moist(i,kk,j,P_QV) + ELSE + qvf = 1. + ENDIF + al(i,kk,j) = (r_d/p1000mb)*(grid%t_2(i,kk,j)+t0)*qvf* & + (((p(i,kk,j)+grid%pb(i,kk,j))/p1000mb)**cvpm) - grid%alb(i,kk,j) + + ! Now, integrate down the column to compute the pressure perturbation, and diagnose the two + ! inverse density fields (total and perturbation). + + DO kk=kpe-2,1,-1 + k = kk + 1 + qvf1 = 0.5*(moist(i,kk,j,P_QV)+moist(i,kk+1,j,P_QV)) + qvf2 = 1./(1.+qvf1) + qvf1 = qvf1*qvf2 + p(i,kk,j) = p(i,kk+1,j) - ((ngrid%c1f(k)*grid%Mu_2(i,j)) + qvf1*(ngrid%c1f(k)*grid%Mub(i,j)+ngrid%c2f(k)))/qvf2/ngrid%rdn(kk+1) + IF ( config_flags%use_theta_m == 0) THEN + qvf = 1. + rvovrd*moist(i,kk,j,P_QV) + ELSE + qvf = 1. + ENDIF + al(i,kk,j) = (r_d/p1000mb)*(grid%t_2(i,kk,j)+t0)*qvf* & + (((p(i,kk,j)+grid%pb(i,kk,j))/p1000mb)**cvpm) - grid%alb(i,kk,j) + END DO + + ! This is the hydrostatic equation used in the model after the small timesteps. In + ! the model, grid%al (inverse density) is computed from the geopotential. + + IF (grid%hypsometric_opt == 1) THEN + DO kk = 2,kpe + k = kk - 1 + grid%ph_2(i,kk,j) = grid%ph_2(i,kk-1,j) - & + ngrid%dnw(kk-1) * ( ((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k))+(ngrid%c1h(k)*grid%mu_2(i,j)))*al(i,kk-1,j) & + + (ngrid%c1h(k)*grid%mu_2(i,j))*grid%alb(i,kk-1,j) ) + END DO + + ! Alternative hydrostatic eq.: dZ = -al*p*dLOG(p), where p is dry pressure. + ! Note that al*p approximates Rd*T and dLOG(p) does z. + ! Here T varies mostly linear with z, the first-order integration produces better result. + + ELSE IF (grid%hypsometric_opt == 2) THEN + + grid%ph_2(i,1,j) = grid%phb(i,1,j) + DO k = 2,kpe + pfu = ngrid%c3f(k )*( grid%MUB(i,j)+grid%MU_2(i,j) ) + ngrid%c4f(k ) + ngrid%p_top + pfd = ngrid%c3f(k-1)*( grid%MUB(i,j)+grid%MU_2(i,j) ) + ngrid%c4f(k-1) + ngrid%p_top + phm = ngrid%c3h(k-1)*( grid%MUB(i,j)+grid%MU_2(i,j) ) + ngrid%c4h(k-1) + ngrid%p_top + grid%ph_2(i,k,j) = grid%ph_2(i,k-1,j) + (grid%alb(i,k-1,j)+al(i,k-1,j))*phm*LOG(pfd/pfu) + END DO + + DO k = 1,kpe + grid%ph_2(i,k,j) = grid%ph_2(i,k,j) - grid%phb(i,k,j) + END DO + + END IF + + END DO ! i loop + END DO ! j loop + + DEALLOCATE(p) + DEALLOCATE(al) + + ! Couple the variables moist and t_2, and the newly calculated ph_2 + DO j = MAX(jds,jps),MIN(jde-1,jpe) + DO i = MAX(ids,ips),MIN(ide-1,ipe) + DO k=kps,kpe + grid%ph_2(i,k,j) = grid%ph_2(i,k,j)*((ngrid%c1f(k)*grid%Mub(i,j)+ngrid%c2f(k)) + (ngrid%c1f(k)*grid%Mu_2(i,j))) + END DO + END DO + END DO + DO j = MAX(jds,jps),MIN(jde-1,jpe) + DO i = MAX(ids,ips),MIN(ide-1,ipe) + DO k=kps,kpe-1 + grid%t_2(i,k,j) = grid%t_2(i,k,j)*((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) + moist(i,k,j,P_QV) = moist(i,k,j,P_QV)*((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) + END DO + END DO + END DO + + + END IF + + +#include "HALO_FORCE_DOWN.inc" + + ! code here to interpolate the data into the nested domain +# include "nest_forcedown_interp.inc" + + RETURN + END SUBROUTINE force_domain_em_part2 + +#endif diff --git a/external/RSL_LITE/interp_domain_em_part1.F b/external/RSL_LITE/interp_domain_em_part1.F new file mode 100644 index 0000000000..c7061f745c --- /dev/null +++ b/external/RSL_LITE/interp_domain_em_part1.F @@ -0,0 +1,105 @@ + +!------------------------------------------------------------------ + +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + + SUBROUTINE interp_domain_em_part1 ( grid, intermediate_grid, ngrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & + nest_task_offsets, nest_pes_x, nest_pes_y, which_kid, & + intercomm_active, mpi_comm_to_kid, mpi_comm_to_mom, & + mytask, get_dm_max_halo_width + USE module_timing + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: intermediate_grid + TYPE(domain), POINTER :: ngrid +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + INTEGER iparstrt,jparstrt,sw + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: iids, iide, ijds, ijde, ikds, ikde, & + iims, iime, ijms, ijme, ikms, ikme, & + iips, iipe, ijps, ijpe, ikps, ikpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER icoord, jcoord, idim_cd, jdim_cd, pgr + INTEGER thisdomain_max_halo_width + INTEGER local_comm, myproc, nproc + INTEGER ioffset, ierr + + CALL wrf_get_dm_communicator ( local_comm ) + CALL wrf_get_myproc( myproc ) + CALL wrf_get_nproc( nproc ) + + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + CALL get_ijk_from_grid ( intermediate_grid , & + iids, iide, ijds, ijde, ikds, ikde, & + iims, iime, ijms, ijme, ikms, ikme, & + iips, iipe, ijps, ijpe, ikps, ikpe ) + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + + CALL nl_get_parent_grid_ratio ( ngrid%id, pgr ) + CALL nl_get_i_parent_start ( intermediate_grid%id, iparstrt ) + CALL nl_get_j_parent_start ( intermediate_grid%id, jparstrt ) + CALL nl_get_shw ( intermediate_grid%id, sw ) + icoord = iparstrt - sw + jcoord = jparstrt - sw + idim_cd = iide - iids + 1 + jdim_cd = ijde - ijds + 1 + + nlev = ckde - ckds + 1 + + ! get max_halo_width for parent. It may be smaller if it is moad + CALL get_dm_max_halo_width ( grid%id , thisdomain_max_halo_width ) + + IF ( grid%active_this_task ) THEN +#include "nest_interpdown_pack.inc" + END IF + + ! determine which communicator and offset to use + IF ( intercomm_active( grid%id ) ) THEN ! I am parent + local_comm = mpi_comm_to_kid( which_kid(ngrid%id), grid%id ) + ioffset = nest_task_offsets(ngrid%id) + ELSE IF ( intercomm_active( ngrid%id ) ) THEN ! I am nest + local_comm = mpi_comm_to_mom( ngrid%id ) + ioffset = nest_task_offsets(ngrid%id) + END IF + + IF ( grid%active_this_task .OR. ngrid%active_this_task ) THEN +#ifndef STUBMPI + CALL mpi_comm_rank(local_comm,myproc,ierr) + CALL mpi_comm_size(local_comm,nproc,ierr) +#endif + CALL rsl_lite_bcast_msgs( myproc, nest_pes_x(grid%id)*nest_pes_y(grid%id), & + nest_pes_x(ngrid%id)*nest_pes_y(ngrid%id), & + ioffset, local_comm ) + END IF + + RETURN + END SUBROUTINE interp_domain_em_part1 +#endif diff --git a/external/RSL_LITE/interp_domain_em_part2.F b/external/RSL_LITE/interp_domain_em_part2.F new file mode 100644 index 0000000000..be6a531c4d --- /dev/null +++ b/external/RSL_LITE/interp_domain_em_part2.F @@ -0,0 +1,187 @@ +!------------------------------------------------------------------ + +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + + SUBROUTINE interp_domain_em_part2 ( grid, ngrid, pgrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & + mytask, get_dm_max_halo_width, which_kid + ! push_communicators_for_domain,pop_communicators_for_domain + USE module_comm_nesting_dm, ONLY : halo_interp_down_sub + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: ngrid + TYPE(domain), POINTER :: pgrid !KAL added for vertical nesting +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER myproc + INTEGER ierr + INTEGER thisdomain_max_halo_width + + !KAL variables for vertical nesting + REAL :: p_top_m , p_surf_m , mu_m , hsca_m , pre_c ,pre_n + REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert) :: alt_w_c + REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert+1) :: alt_u_c + REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert) :: alt_w_n + REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert+1) :: alt_u_n + + + !KAL change this for vertical nesting + ! interp_domain_em_part1 packs up the interpolation onto the coarse (vertical) grid + ! therefore the message size is based on the coarse grid number of levels + ! here it is unpacked onto the intermediate grid + CALL get_ijk_from_grid ( pgrid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + !KAL this is the original WRF code + !CALL get_ijk_from_grid ( grid , & + ! cids, cide, cjds, cjde, ckds, ckde, & + ! cims, cime, cjms, cjme, ckms, ckme, & + ! cips, cipe, cjps, cjpe, ckps, ckpe ) + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + + nlev = ckde - ckds + 1 + + CALL get_dm_max_halo_width ( ngrid%id , thisdomain_max_halo_width ) + +#include "nest_interpdown_unpack.inc" + + +if (ngrid%vert_refine_method .NE. 0) then + + !KAL calculating the vertical coordinate for parent and nest grid (code from ndown) + ! assume that the parent and nest have the same p_top value (as in ndown) + +!KAL ckde is equal to e_vert of the coarse grid. There are e_vert-1 u points. The coarse 1D grid here is e_vert+1, +! so it is the e_vert-1 points from the coarse grid, plus a surface point plus a top point. Extrapolation coefficients +! are used to get the surface and top points to fill out the pro_u_c 1D array of u values from the coarse grid. + + hsca_m = 6.7 !KAL scale height of the atmosphere + p_top_m = ngrid%p_top + p_surf_m = 1.e5 + mu_m = p_surf_m - p_top_m +! parent + do k = 1,ckde + pre_c = mu_m * pgrid%c3f(k) + p_top_m + pgrid%c4f(k) + alt_w_c(k) = -hsca_m * alog(pre_c/p_surf_m) + enddo + do k = 1,ckde-1 + pre_c = mu_m * pgrid%c3h(k) + p_top_m + pgrid%c4h(k) + alt_u_c(k+1) = -hsca_m * alog(pre_c/p_surf_m) + enddo + alt_u_c(1) = alt_w_c(1) + alt_u_c(ckde+1) = alt_w_c(ckde) +! nest + do k = 1,nkde + pre_n = mu_m * ngrid%c3f(k) + p_top_m + ngrid%c4f(k) + alt_w_n(k) = -hsca_m * alog(pre_n/p_surf_m) + enddo + do k = 1,nkde-1 + pre_n = mu_m * ngrid%c3h(k) + p_top_m + ngrid%c4h(k) + alt_u_n(k+1) = -hsca_m * alog(pre_n/p_surf_m) + enddo + alt_u_n(1) = alt_w_n(1) + alt_u_n(nkde+1) = alt_w_n(nkde) +endif + + + + !KAL added this call for vertical nesting (return coarse grid dimensions to intended values) + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + + CALL get_ijk_from_grid ( grid , & + ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe ) + + +if (ngrid%vert_refine_method .NE. 0) then + +!KAL added this code (the include file) for the vertical nesting +#include "nest_interpdown_interp_vert.inc" + + + !KAL finish off the 1-D variables (t_base, u_base, v_base, qv_base, and z_base) (move this out of here if alt_u_c and alt_u_n are calculated elsewhere) + CALL vert_interp_vert_nesting_1d ( & + ngrid%t_base, & ! CD field + ids, ide, kds, kde, jds, jde, & ! CD dims + ims, ime, kms, kme, jms, jme, & ! CD dims + ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims + pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid + pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants + alt_u_c, alt_u_n) ! coordinates for parent and nest + CALL vert_interp_vert_nesting_1d ( & + ngrid%u_base, & ! CD field + ids, ide, kds, kde, jds, jde, & ! CD dims + ims, ime, kms, kme, jms, jme, & ! CD dims + ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims + pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid + pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants + alt_u_c, alt_u_n) ! coordinates for parent and nest + CALL vert_interp_vert_nesting_1d ( & + ngrid%v_base, & ! CD field + ids, ide, kds, kde, jds, jde, & ! CD dims + ims, ime, kms, kme, jms, jme, & ! CD dims + ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims + pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid + pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants + alt_u_c, alt_u_n) ! coordinates for parent and nest + CALL vert_interp_vert_nesting_1d ( & + ngrid%qv_base, & ! CD field + ids, ide, kds, kde, jds, jde, & ! CD dims + ims, ime, kms, kme, jms, jme, & ! CD dims + ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims + pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid + pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants + alt_u_c, alt_u_n) ! coordinates for parent and nest + CALL vert_interp_vert_nesting_1d ( & + ngrid%z_base, & ! CD field + ids, ide, kds, kde, jds, jde, & ! CD dims + ims, ime, kms, kme, jms, jme, & ! CD dims + ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims + pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid + pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants + alt_u_c, alt_u_n) ! coordinates for parent and nest + +endif + + CALL push_communicators_for_domain( grid%id ) + +#include "HALO_INTERP_DOWN.inc" + + CALL pop_communicators_for_domain + + RETURN + END SUBROUTINE interp_domain_em_part2 +#endif diff --git a/external/RSL_LITE/interp_domain_em_part3.F b/external/RSL_LITE/interp_domain_em_part3.F new file mode 100644 index 0000000000..9367e23b9b --- /dev/null +++ b/external/RSL_LITE/interp_domain_em_part3.F @@ -0,0 +1,59 @@ +!------------------------------------------------------------------ + +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + + SUBROUTINE interp_domain_em_part3 ( grid, ngrid, pgrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & + mytask, get_dm_max_halo_width, which_kid + ! push_communicators_for_domain,pop_communicators_for_domain + USE module_comm_nesting_dm, ONLY : halo_interp_down_sub + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: ngrid + TYPE(domain), POINTER :: pgrid !KAL added for vertical nesting +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER myproc + INTEGER ierr + INTEGER thisdomain_max_halo_width + + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + +# include "nest_interpdown_interp.inc" + + RETURN + END SUBROUTINE interp_domain_em_part3 +#endif diff --git a/external/RSL_LITE/interp_domain_em_small.F b/external/RSL_LITE/interp_domain_em_small.F new file mode 100644 index 0000000000..cfb175a95f --- /dev/null +++ b/external/RSL_LITE/interp_domain_em_small.F @@ -0,0 +1,411 @@ + +!------------------------------------------------------------------ + +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + + SUBROUTINE interp_domain_em_small_part1 ( grid, intermediate_grid, ngrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_comm_dm, ONLY: halo_em_horiz_interp_sub + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & + mytask, get_dm_max_halo_width, & + nest_task_offsets, mpi_comm_to_kid, mpi_comm_to_mom, & + which_kid, nest_pes_x, nest_pes_y, intercomm_active + USE module_timing + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: intermediate_grid + TYPE(domain), POINTER :: ngrid +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + INTEGER iparstrt,jparstrt,sw + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: iids, iide, ijds, ijde, ikds, ikde, & + iims, iime, ijms, ijme, ikms, ikme, & + iips, iipe, ijps, ijpe, ikps, ikpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER icoord, jcoord, idim_cd, jdim_cd, pgr + INTEGER thisdomain_max_halo_width + INTEGER local_comm, myproc, nproc + INTEGER ioffset + + CALL wrf_get_dm_communicator ( local_comm ) + CALL wrf_get_myproc( myproc ) + CALL wrf_get_nproc( nproc ) + + CALL get_ijk_from_grid ( grid , & + ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe ) +#ifdef DM_PARALLEL +# include "HALO_EM_HORIZ_INTERP.inc" +#endif + + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + CALL get_ijk_from_grid ( intermediate_grid , & + iids, iide, ijds, ijde, ikds, ikde, & + iims, iime, ijms, ijme, ikms, ikme, & + iips, iipe, ijps, ijpe, ikps, ikpe ) + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + + CALL nl_get_parent_grid_ratio ( ngrid%id, pgr ) + CALL nl_get_i_parent_start ( intermediate_grid%id, iparstrt ) + CALL nl_get_j_parent_start ( intermediate_grid%id, jparstrt ) + CALL nl_get_shw ( intermediate_grid%id, sw ) + icoord = iparstrt - sw + jcoord = jparstrt - sw + idim_cd = iide - iids + 1 + jdim_cd = ijde - ijds + 1 + + nlev = ckde - ckds + 1 + + ! get max_halo_width for parent. It may be smaller if it is moad + CALL get_dm_max_halo_width ( grid%id , thisdomain_max_halo_width ) + + ! How many 3d arrays, so far just 3d theta-300 and geopotential perturbation, + ! and the 2d topo elevation, three max press/temp/height fields, and three + ! min press/temp/height fields. + + msize = ( 2 )* nlev + 7 + +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, calling rsl_lite_to_child') + CALL rsl_lite_to_child_info( local_communicator, msize*RWORDSIZE & + ,cips,cipe,cjps,cjpe & + ,iids,iide,ijds,ijde & + ,nids,nide,njds,njde & + ,pgr , sw & + ,ntasks_x,ntasks_y & + ,thisdomain_max_halo_width & + ,icoord,jcoord & + ,idim_cd,jdim_cd & + ,pig,pjg,retval ) +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, back from rsl_lite_to_child') + DO while ( retval .eq. 1 ) + IF ( SIZE(grid%ph_2) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ph_2') + DO k = ckds,ckde + xv(k)= grid%ph_2(pig,k,pjg) + END DO + CALL rsl_lite_to_child_msg(((ckde)-(ckds)+1)*RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%t_2) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, t_2') + DO k = ckds,(ckde-1) + xv(k)= grid%t_2(pig,k,pjg) + END DO + CALL rsl_lite_to_child_msg((((ckde-1))-(ckds)+1)*RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%ht) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ht') + xv(1)= grid%ht(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%t_max_p) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, t_max_p') + xv(1)= grid%t_max_p(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%ght_max_p) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ght_max_p') + xv(1)= grid%ght_max_p(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%max_p) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, max_p') + xv(1)= grid%max_p(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%t_min_p) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, t_min_p') + xv(1)= grid%t_min_p(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%ght_min_p) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ght_min_p') + xv(1)= grid%ght_min_p(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%min_p) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, min_p') + xv(1)= grid%min_p(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, calling rsl_lite_to_child_info') + CALL rsl_lite_to_child_info( local_communicator, msize*RWORDSIZE & + ,cips,cipe,cjps,cjpe & + ,iids,iide,ijds,ijde & + ,nids,nide,njds,njde & + ,pgr , sw & + ,ntasks_x,ntasks_y & + ,thisdomain_max_halo_width & + ,icoord,jcoord & + ,idim_cd,jdim_cd & + ,pig,pjg,retval ) +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, back from rsl_lite_to_child_info') + END DO + + ! determine which communicator and offset to use + IF ( intercomm_active( grid%id ) ) THEN ! I am parent + local_comm = mpi_comm_to_kid( which_kid(ngrid%id), grid%id ) + ioffset = nest_task_offsets(ngrid%id) + ELSE IF ( intercomm_active( ngrid%id ) ) THEN ! I am nest + local_comm = mpi_comm_to_mom( ngrid%id ) + ioffset = nest_task_offsets(ngrid%id) + END IF + +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, calling rsl_lite_bcast') + CALL rsl_lite_bcast_msgs( myproc, nest_pes_x(grid%id)*nest_pes_y(grid%id), & + nest_pes_x(ngrid%id)*nest_pes_y(ngrid%id), & + ioffset, local_comm ) +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, back from rsl_lite_bcast') + + RETURN + END SUBROUTINE interp_domain_em_small_part1 + +!------------------------------------------------------------------ + + SUBROUTINE interp_domain_em_small_part2 ( grid, ngrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & + mytask, get_dm_max_halo_width + USE module_comm_nesting_dm, ONLY : halo_interp_down_sub + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: ngrid +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER myproc + INTEGER ierr + INTEGER thisdomain_max_halo_width + + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + + nlev = ckde - ckds + 1 + + CALL get_dm_max_halo_width ( ngrid%id , thisdomain_max_halo_width ) + + CALL rsl_lite_from_parent_info(pig,pjg,retval) + + DO while ( retval .eq. 1 ) + + IF ( SIZE(grid%ph_2) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(((ckde)-(ckds)+1)*RWORDSIZE,xv) + DO k = ckds,ckde + grid%ph_2(pig,k,pjg) = xv(k) + END DO + END IF + + IF ( SIZE(grid%t_2) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg((((ckde-1))-(ckds)+1)*RWORDSIZE,xv) + DO k = ckds,(ckde-1) + grid%t_2(pig,k,pjg) = xv(k) + END DO + END IF + + IF ( SIZE(grid%ht) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%ht(pig,pjg) = xv(1) + END IF + + IF ( SIZE(grid%t_max_p) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%t_max_p(pig,pjg) = xv(1) + END IF + + IF ( SIZE(grid%ght_max_p) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%ght_max_p(pig,pjg) = xv(1) + END IF + + IF ( SIZE(grid%max_p) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%max_p(pig,pjg) = xv(1) + END IF + + IF ( SIZE(grid%t_min_p) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%t_min_p(pig,pjg) = xv(1) + END IF + + IF ( SIZE(grid%ght_min_p) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%ght_min_p(pig,pjg) = xv(1) + END IF + + IF ( SIZE(grid%min_p) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%min_p(pig,pjg) = xv(1) + END IF + + CALL rsl_lite_from_parent_info(pig,pjg,retval) + + END DO + + CALL get_ijk_from_grid ( grid , & + ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe ) + +#include "HALO_INTERP_DOWN.inc" + + CALL interp_fcn_bl ( grid%ph_2, & + cids, cide, ckds, ckde, cjds, cjde, & + cims, cime, ckms, ckme, cjms, cjme, & + cips, cipe, ckps, MIN( ckde, ckpe ), cjps, cjpe, & + ngrid%ph_2, & + nids, nide, nkds, nkde, njds, njde, & + nims, nime, nkms, nkme, njms, njme, & + nips, nipe, nkps, MIN( nkde, nkpe ), njps, njpe, & + config_flags%shw, ngrid%imask_nostag, & + .FALSE., .FALSE., & + ngrid%i_parent_start, ngrid%j_parent_start, & + ngrid%parent_grid_ratio, ngrid%parent_grid_ratio, & + grid%ht, ngrid%ht, & + grid%t_max_p, ngrid%t_max_p, & + grid%ght_max_p, ngrid%ght_max_p, & + grid%max_p, ngrid%max_p, & + grid%t_min_p, ngrid%t_min_p, & + grid%ght_min_p, ngrid%ght_min_p, & + grid%min_p, ngrid%min_p, & + ngrid%znw, ngrid%p_top ) + + CALL interp_fcn_bl ( grid%t_2, & + cids, cide, ckds, ckde, cjds, cjde, & + cims, cime, ckms, ckme, cjms, cjme, & + cips, cipe, ckps, MIN( (ckde-1), ckpe ), cjps, cjpe, & + ngrid%t_2, & + nids, nide, nkds, nkde, njds, njde, & + nims, nime, nkms, nkme, njms, njme, & + nips, nipe, nkps, MIN( (nkde-1), nkpe ), njps, njpe, & + config_flags%shw, ngrid%imask_nostag, & + .FALSE., .FALSE., & + ngrid%i_parent_start, ngrid%j_parent_start, & + ngrid%parent_grid_ratio, ngrid%parent_grid_ratio, & + grid%ht, ngrid%ht, & + grid%t_max_p, ngrid%t_max_p, & + grid%ght_max_p, ngrid%ght_max_p, & + grid%max_p, ngrid%max_p, & + grid%t_min_p, ngrid%t_min_p, & + grid%ght_min_p, ngrid%ght_min_p, & + grid%min_p, ngrid%min_p, & + ngrid%znu, ngrid%p_top ) + + RETURN + END SUBROUTINE interp_domain_em_small_part2 + +!------------------------------------------------------------------ + + SUBROUTINE feedback_nest_prep ( grid, config_flags & +! +#include "dummy_new_args.inc" +! +) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, mytask !, & + !push_communicators_for_domain, pop_communicators_for_domain + USE module_comm_nesting_dm, ONLY : halo_interp_up_sub + IMPLICIT NONE +! + TYPE(domain), TARGET :: grid ! name of the grid being dereferenced (must be "grid") + TYPE (grid_config_rec_type) :: config_flags ! configureation flags, has vertical dim of + ! soil temp, moisture, etc., has vertical dim + ! of soil categories +#include "dummy_new_decl.inc" + + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER :: idum1, idum2 + + + CALL get_ijk_from_grid ( grid , & + ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe ) + + IF ( grid%active_this_task ) THEN + CALL push_communicators_for_domain( grid%id ) + +#ifdef DM_PARALLEL +#include "HALO_INTERP_UP.inc" +#endif + + CALL pop_communicators_for_domain + END IF + + END SUBROUTINE feedback_nest_prep + +!------------------------------------------------------------------ + +#endif diff --git a/external/RSL_LITE/makefile b/external/RSL_LITE/makefile index 5d1139fe4c..bcd3e2d05d 100644 --- a/external/RSL_LITE/makefile +++ b/external/RSL_LITE/makefile @@ -9,6 +9,19 @@ CFLAGS = $(PLUSFLAG) CPPFLAGS = FFLAGS = +# These .F files are compiled in frame but leave behind auto-gen files here +F_FILES = \ + feedback_domain_em_part1.F \ + feedback_domain_em_part2.F \ + force_domain_em_part2.F \ + interp_domain_em_part1.F \ + interp_domain_em_part2.F \ + interp_domain_em_part3.F \ + interp_domain_em_small.F + +# These will be the files we need to clean up +F90_FILES = $(foreach file,$(F_FILES),$(patsubst %.F,%.f90,$(file))) + .SUFFIXES: .F90 .F .f .o .code @@ -51,7 +64,7 @@ f_xpose.o: f_xpose.F90 f_pack.o $(FC) -o $@ $(FFLAGS) -c f_xpose.f clean : - @/bin/rm -f *.f *.o *.mod *.obj *.i + @/bin/rm -f *.f *.o *.mod *.obj *.i $(F90_FILES) superclean : clean @/bin/rm -f *.a diff --git a/external/RSL_LITE/module_dm.F b/external/RSL_LITE/module_dm.F index 5c9bb62a67..ea166ae384 100644 --- a/external/RSL_LITE/module_dm.F +++ b/external/RSL_LITE/module_dm.F @@ -3918,1320 +3918,6 @@ SUBROUTINE wrf_dm_nestexchange_init END SUBROUTINE wrf_dm_nestexchange_init -!------------------------------------------------------------------ - -#if ( EM_CORE == 1 && DA_CORE != 1 ) - -!------------------------------------------------------------------ - - SUBROUTINE force_domain_em_part2 ( grid, ngrid, pgrid, config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, local_communicator, mytask, & - nest_pes_x, nest_pes_y ! , & - !push_communicators_for_domain,pop_communicators_for_domain - USE module_comm_nesting_dm, ONLY : halo_force_down_sub - USE module_model_constants - IMPLICIT NONE -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: ngrid - TYPE(domain), POINTER :: pgrid !KAL added for vertical nesting -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k,kk - TYPE (grid_config_rec_type) :: config_flags - REAL xv(2000) - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - INTEGER :: ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7,itrace - REAL dummy_xs, dummy_xe, dummy_ys, dummy_ye - - !KAL variables for vertical nesting - REAL :: p_top_m , p_surf_m , mu_m , hsca_m , pre_c ,pre_n - REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert) :: alt_w_c - REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert+1) :: alt_u_c - REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert) :: alt_w_n - REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert+1) :: alt_u_n - - REAL, DIMENSION(:,:,:), ALLOCATABLE :: p, al - REAL :: pfu, pfd, phm, temp, qvf, qvf1, qvf2 - - !KAL change this for vertical nesting - ! force_domain_em_part1 packs up the interpolation onto the coarse (vertical) grid - ! therefore the message size is based on the coarse grid number of levels - ! here it is unpacked onto the intermediate grid - CALL get_ijk_from_grid ( pgrid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - - !KAL this is the original WRF code - !CALL get_ijk_from_grid ( grid , & - ! cids, cide, cjds, cjde, ckds, ckde, & - ! cims, cime, cjms, cjme, ckms, ckme, & - ! cips, cipe, cjps, cjpe, ckps, ckpe ) - CALL get_ijk_from_grid ( ngrid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - - nlev = ckde - ckds + 1 - -#include "nest_interpdown_unpack.inc" - -if (ngrid%vert_refine_method .NE. 0) then - - !KAL calculating the vertical coordinate for parent and nest grid (code from ndown) - ! assume that the parent and nest have the same p_top value (as in ndown) - -!KAL ckde is equal to e_vert of the coarse grid. There are e_vert-1 u points. The coarse 1D grid here is e_vert+1, -! so it is the e_vert-1 points from the coarse grid, plus a surface point plus a top point. Extrapolation coefficients -! are used to get the surface and top points to fill out the pro_u_c 1D array of u values from the coarse grid. - - hsca_m = 6.7 !KAL scale height of the atmosphere - p_top_m = ngrid%p_top - p_surf_m = 1.e5 - mu_m = p_surf_m - p_top_m -! parent - do k = 1,ckde - pre_c = mu_m * pgrid%c3f(k) + p_top_m + pgrid%c4f(k) - alt_w_c(k) = -hsca_m * alog(pre_c/p_surf_m) - enddo - do k = 1,ckde-1 - pre_c = mu_m * pgrid%c3h(k) + p_top_m + pgrid%c4h(k) - alt_u_c(k+1) = -hsca_m * alog(pre_c/p_surf_m) - enddo - alt_u_c(1) = alt_w_c(1) - alt_u_c(ckde+1) = alt_w_c(ckde) -! nest - do k = 1,nkde - pre_n = mu_m * ngrid%c3f(k) + p_top_m + ngrid%c4f(k) - alt_w_n(k) = -hsca_m * alog(pre_n/p_surf_m) - enddo - do k = 1,nkde-1 - pre_n = mu_m * ngrid%c3h(k) + p_top_m + ngrid%c4h(k) - alt_u_n(k+1) = -hsca_m * alog(pre_n/p_surf_m) - enddo - alt_u_n(1) = alt_w_n(1) - alt_u_n(nkde+1) = alt_w_n(nkde) - -endif - - !KAL added this call for vertical nesting (return coarse grid dimensions to intended values) - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - - CALL get_ijk_from_grid ( grid , & - ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe ) - - ! Vertical refinement is turned on. - - IF (ngrid%vert_refine_method .NE. 0) THEN - -#include "nest_forcedown_interp_vert.inc" - - IF ( ngrid%this_is_an_ideal_run ) THEN - IF ( SIZE( grid%t_init, 1 ) * SIZE( grid%t_init, 3 ) .GT. 1 ) THEN - CALL vert_interp_vert_nesting( grid%t_init, & !CD field - ids, ide, kds, kde, jds, jde, & !CD dims - ims, ime, kms, kme, jms, jme, & !CD dims - ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & !CD dims - pgrid%s_vert, pgrid%e_vert, & !vertical dimension of the parent grid - pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & !coarse grid extrapolation constants - alt_u_c, alt_u_n ) !coordinates for parent and nest - END IF ! Check t_init is a fully allocated 3d array. - END IF ! only for ideal runs - - - ! Rebalance the grid on the intermediate grid. The intermediate grid has the horizontal - ! resolution of the parent grid, but at this point has been interpolated in the vertical - ! to the resolution of the nest. The base state (phb, pb, etc) from the parent grid is - ! unpacked onto the intermediate grid every time this subroutine is called. We need the - ! base state of the nest, so it is recalculated here. - - ! Additionally, we do not need to vertically interpolate the entire intermediate grid - ! above, just the points that contribute to the boundary forcing. - - ! Base state potential temperature and inverse density (alpha = 1/rho) from - ! the half eta levels and the base-profile surface pressure. Compute 1/rho - ! from equation of state. The potential temperature is a perturbation from t0. - - ! Uncouple the variables moist and t_2 that are used to calculate ph_2 - - DO j = MAX(jds,jps),MIN(jde-1,jpe) - DO i = MAX(ids,ips),MIN(ide-1,ipe) - DO k=kds,kde-1 - grid%t_2(i,k,j) = grid%t_2(i,k,j)/((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) - moist(i,k,j,P_QV) = moist(i,k,j,P_QV)/((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) - END DO - END DO - END DO - - DO j = MAX(jds,jps),MIN(jde-1,jpe) - DO i = MAX(ids,ips),MIN(ide-1,ipe) - - DO k = 1, kpe-1 - grid%pb(i,k,j) = ngrid%c3h(k) * grid%mub(i,j) + ngrid%c4h(k) + ngrid%p_top - - ! If this is a real run, recalc t_init. - - IF ( .NOT. ngrid%this_is_an_ideal_run ) THEN - temp = MAX ( ngrid%tiso, ngrid%t00 + ngrid%tlp*LOG(grid%pb(i,k,j)/ngrid%p00) ) - IF ( grid%pb(i,k,j) .LT. ngrid%p_strat ) THEN - temp = ngrid%tiso + ngrid%tlp_strat * LOG ( grid%pb(i,k,j)/ngrid%p_strat ) - END IF - grid%t_init(i,k,j) = temp*(ngrid%p00/grid%pb(i,k,j))**(r_d/cp) - t0 - END IF - grid%alb(i,k,j) = (r_d/p1000mb)*(grid%t_init(i,k,j)+t0)*(grid%pb(i,k,j)/p1000mb)**cvpm - END DO - - ! Integrate base geopotential, starting at terrain elevation. This assures that - ! the base state is in exact hydrostatic balance with respect to the model equations. - ! This field is on full levels. - - grid%phb(i,1,j) = grid%ht(i,j) * g - IF (grid%hypsometric_opt == 1) THEN - DO kk = 2,kpe - k = kk - 1 - grid%phb(i,kk,j) = grid%phb(i,k,j) - ngrid%dnw(k)*(ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k))*grid%alb(i,k,j) - END DO - ELSE IF (grid%hypsometric_opt == 2) THEN - DO k = 2,kpe - pfu = ngrid%c3f(k )*grid%MUB(i,j) + ngrid%c4f(k ) + ngrid%p_top - pfd = ngrid%c3f(k-1)*grid%MUB(i,j) + ngrid%c4f(k-1) + ngrid%p_top - phm = ngrid%c3h(k-1)*grid%MUB(i,j) + ngrid%c4h(k-1) + ngrid%p_top - grid%phb(i,k,j) = grid%phb(i,k-1,j) + grid%alb(i,k-1,j)*phm*LOG(pfd/pfu) - END DO - ELSE - CALL wrf_error_fatal( 'module_dm: hypsometric_opt should be 1 or 2' ) - END IF ! which hypsometric option - END DO ! i loop - END DO ! j loop - ! Perturbation fields - ALLOCATE( p (ips:ipe, kps:kpe, jps:jpe) ) - ALLOCATE( al(ips:ipe, kps:kpe, jps:jpe) ) - DO j = MAX(jds,jps),MIN(jde-1,jpe) - DO i = MAX(ids,ips),MIN(ide-1,ipe) - ! Integrate the hydrostatic equation (from the RHS of the bigstep vertical momentum - ! equation) down from the top to get the pressure perturbation. First get the pressure - ! perturbation, moisture, and inverse density (total and perturbation) at the top-most level. - - kk = kpe-1 - k = kk+1 - - qvf1 = 0.5*(moist(i,kk,j,P_QV)+moist(i,kk,j,P_QV)) - qvf2 = 1./(1.+qvf1) - qvf1 = qvf1*qvf2 - - p(i,kk,j) = - 0.5*((ngrid%c1f(k)*grid%Mu_2(i,j))+qvf1*(ngrid%c1f(k)*grid%Mub(i,j)+ngrid%c2f(k)))/ngrid%rdnw(kk)/qvf2 - IF ( config_flags%use_theta_m == 0) THEN - qvf = 1. + rvovrd*moist(i,kk,j,P_QV) - ELSE - qvf = 1. - ENDIF - al(i,kk,j) = (r_d/p1000mb)*(grid%t_2(i,kk,j)+t0)*qvf* & - (((p(i,kk,j)+grid%pb(i,kk,j))/p1000mb)**cvpm) - grid%alb(i,kk,j) - - ! Now, integrate down the column to compute the pressure perturbation, and diagnose the two - ! inverse density fields (total and perturbation). - - DO kk=kpe-2,1,-1 - k = kk + 1 - qvf1 = 0.5*(moist(i,kk,j,P_QV)+moist(i,kk+1,j,P_QV)) - qvf2 = 1./(1.+qvf1) - qvf1 = qvf1*qvf2 - p(i,kk,j) = p(i,kk+1,j) - ((ngrid%c1f(k)*grid%Mu_2(i,j)) + qvf1*(ngrid%c1f(k)*grid%Mub(i,j)+ngrid%c2f(k)))/qvf2/ngrid%rdn(kk+1) - IF ( config_flags%use_theta_m == 0) THEN - qvf = 1. + rvovrd*moist(i,kk,j,P_QV) - ELSE - qvf = 1. - ENDIF - al(i,kk,j) = (r_d/p1000mb)*(grid%t_2(i,kk,j)+t0)*qvf* & - (((p(i,kk,j)+grid%pb(i,kk,j))/p1000mb)**cvpm) - grid%alb(i,kk,j) - END DO - - ! This is the hydrostatic equation used in the model after the small timesteps. In - ! the model, grid%al (inverse density) is computed from the geopotential. - - IF (grid%hypsometric_opt == 1) THEN - DO kk = 2,kpe - k = kk - 1 - grid%ph_2(i,kk,j) = grid%ph_2(i,kk-1,j) - & - ngrid%dnw(kk-1) * ( ((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k))+(ngrid%c1h(k)*grid%mu_2(i,j)))*al(i,kk-1,j) & - + (ngrid%c1h(k)*grid%mu_2(i,j))*grid%alb(i,kk-1,j) ) - END DO - - ! Alternative hydrostatic eq.: dZ = -al*p*dLOG(p), where p is dry pressure. - ! Note that al*p approximates Rd*T and dLOG(p) does z. - ! Here T varies mostly linear with z, the first-order integration produces better result. - - ELSE IF (grid%hypsometric_opt == 2) THEN - - grid%ph_2(i,1,j) = grid%phb(i,1,j) - DO k = 2,kpe - pfu = ngrid%c3f(k )*( grid%MUB(i,j)+grid%MU_2(i,j) ) + ngrid%c4f(k ) + ngrid%p_top - pfd = ngrid%c3f(k-1)*( grid%MUB(i,j)+grid%MU_2(i,j) ) + ngrid%c4f(k-1) + ngrid%p_top - phm = ngrid%c3h(k-1)*( grid%MUB(i,j)+grid%MU_2(i,j) ) + ngrid%c4h(k-1) + ngrid%p_top - grid%ph_2(i,k,j) = grid%ph_2(i,k-1,j) + (grid%alb(i,k-1,j)+al(i,k-1,j))*phm*LOG(pfd/pfu) - END DO - - DO k = 1,kpe - grid%ph_2(i,k,j) = grid%ph_2(i,k,j) - grid%phb(i,k,j) - END DO - - END IF - - END DO ! i loop - END DO ! j loop - - DEALLOCATE(p) - DEALLOCATE(al) - - ! Couple the variables moist and t_2, and the newly calculated ph_2 - DO j = MAX(jds,jps),MIN(jde-1,jpe) - DO i = MAX(ids,ips),MIN(ide-1,ipe) - DO k=kps,kpe - grid%ph_2(i,k,j) = grid%ph_2(i,k,j)*((ngrid%c1f(k)*grid%Mub(i,j)+ngrid%c2f(k)) + (ngrid%c1f(k)*grid%Mu_2(i,j))) - END DO - END DO - END DO - DO j = MAX(jds,jps),MIN(jde-1,jpe) - DO i = MAX(ids,ips),MIN(ide-1,ipe) - DO k=kps,kpe-1 - grid%t_2(i,k,j) = grid%t_2(i,k,j)*((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) - moist(i,k,j,P_QV) = moist(i,k,j,P_QV)*((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) - END DO - END DO - END DO - - - END IF - - -#include "HALO_FORCE_DOWN.inc" - - ! code here to interpolate the data into the nested domain -# include "nest_forcedown_interp.inc" - - RETURN - END SUBROUTINE force_domain_em_part2 - -!------------------------------------------------------------------ - - SUBROUTINE interp_domain_em_part1 ( grid, intermediate_grid, ngrid, config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & - nest_task_offsets, nest_pes_x, nest_pes_y, which_kid, & - intercomm_active, mpi_comm_to_kid, mpi_comm_to_mom, & - mytask, get_dm_max_halo_width - USE module_timing - IMPLICIT NONE -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: intermediate_grid - TYPE(domain), POINTER :: ngrid -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k - INTEGER iparstrt,jparstrt,sw - TYPE (grid_config_rec_type) :: config_flags - REAL xv(2000) - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: iids, iide, ijds, ijde, ikds, ikde, & - iims, iime, ijms, ijme, ikms, ikme, & - iips, iipe, ijps, ijpe, ikps, ikpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER icoord, jcoord, idim_cd, jdim_cd, pgr - INTEGER thisdomain_max_halo_width - INTEGER local_comm, myproc, nproc - INTEGER ioffset, ierr - - CALL wrf_get_dm_communicator ( local_comm ) - CALL wrf_get_myproc( myproc ) - CALL wrf_get_nproc( nproc ) - - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - CALL get_ijk_from_grid ( intermediate_grid , & - iids, iide, ijds, ijde, ikds, ikde, & - iims, iime, ijms, ijme, ikms, ikme, & - iips, iipe, ijps, ijpe, ikps, ikpe ) - CALL get_ijk_from_grid ( ngrid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - - CALL nl_get_parent_grid_ratio ( ngrid%id, pgr ) - CALL nl_get_i_parent_start ( intermediate_grid%id, iparstrt ) - CALL nl_get_j_parent_start ( intermediate_grid%id, jparstrt ) - CALL nl_get_shw ( intermediate_grid%id, sw ) - icoord = iparstrt - sw - jcoord = jparstrt - sw - idim_cd = iide - iids + 1 - jdim_cd = ijde - ijds + 1 - - nlev = ckde - ckds + 1 - - ! get max_halo_width for parent. It may be smaller if it is moad - CALL get_dm_max_halo_width ( grid%id , thisdomain_max_halo_width ) - - IF ( grid%active_this_task ) THEN -#include "nest_interpdown_pack.inc" - END IF - - ! determine which communicator and offset to use - IF ( intercomm_active( grid%id ) ) THEN ! I am parent - local_comm = mpi_comm_to_kid( which_kid(ngrid%id), grid%id ) - ioffset = nest_task_offsets(ngrid%id) - ELSE IF ( intercomm_active( ngrid%id ) ) THEN ! I am nest - local_comm = mpi_comm_to_mom( ngrid%id ) - ioffset = nest_task_offsets(ngrid%id) - END IF - - IF ( grid%active_this_task .OR. ngrid%active_this_task ) THEN -#ifndef STUBMPI - CALL mpi_comm_rank(local_comm,myproc,ierr) - CALL mpi_comm_size(local_comm,nproc,ierr) -#endif - CALL rsl_lite_bcast_msgs( myproc, nest_pes_x(grid%id)*nest_pes_y(grid%id), & - nest_pes_x(ngrid%id)*nest_pes_y(ngrid%id), & - ioffset, local_comm ) - END IF - - RETURN - END SUBROUTINE interp_domain_em_part1 - -!------------------------------------------------------------------ - - SUBROUTINE interp_domain_em_part2 ( grid, ngrid, pgrid, config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & - mytask, get_dm_max_halo_width, which_kid - ! push_communicators_for_domain,pop_communicators_for_domain - USE module_comm_nesting_dm, ONLY : halo_interp_down_sub - IMPLICIT NONE -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: ngrid - TYPE(domain), POINTER :: pgrid !KAL added for vertical nesting -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k - TYPE (grid_config_rec_type) :: config_flags - REAL xv(2000) - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - INTEGER :: ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER myproc - INTEGER ierr - INTEGER thisdomain_max_halo_width - - !KAL variables for vertical nesting - REAL :: p_top_m , p_surf_m , mu_m , hsca_m , pre_c ,pre_n - REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert) :: alt_w_c - REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert+1) :: alt_u_c - REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert) :: alt_w_n - REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert+1) :: alt_u_n - - - !KAL change this for vertical nesting - ! interp_domain_em_part1 packs up the interpolation onto the coarse (vertical) grid - ! therefore the message size is based on the coarse grid number of levels - ! here it is unpacked onto the intermediate grid - CALL get_ijk_from_grid ( pgrid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - !KAL this is the original WRF code - !CALL get_ijk_from_grid ( grid , & - ! cids, cide, cjds, cjde, ckds, ckde, & - ! cims, cime, cjms, cjme, ckms, ckme, & - ! cips, cipe, cjps, cjpe, ckps, ckpe ) - CALL get_ijk_from_grid ( ngrid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - - nlev = ckde - ckds + 1 - - CALL get_dm_max_halo_width ( ngrid%id , thisdomain_max_halo_width ) - -#include "nest_interpdown_unpack.inc" - - -if (ngrid%vert_refine_method .NE. 0) then - - !KAL calculating the vertical coordinate for parent and nest grid (code from ndown) - ! assume that the parent and nest have the same p_top value (as in ndown) - -!KAL ckde is equal to e_vert of the coarse grid. There are e_vert-1 u points. The coarse 1D grid here is e_vert+1, -! so it is the e_vert-1 points from the coarse grid, plus a surface point plus a top point. Extrapolation coefficients -! are used to get the surface and top points to fill out the pro_u_c 1D array of u values from the coarse grid. - - hsca_m = 6.7 !KAL scale height of the atmosphere - p_top_m = ngrid%p_top - p_surf_m = 1.e5 - mu_m = p_surf_m - p_top_m -! parent - do k = 1,ckde - pre_c = mu_m * pgrid%c3f(k) + p_top_m + pgrid%c4f(k) - alt_w_c(k) = -hsca_m * alog(pre_c/p_surf_m) - enddo - do k = 1,ckde-1 - pre_c = mu_m * pgrid%c3h(k) + p_top_m + pgrid%c4h(k) - alt_u_c(k+1) = -hsca_m * alog(pre_c/p_surf_m) - enddo - alt_u_c(1) = alt_w_c(1) - alt_u_c(ckde+1) = alt_w_c(ckde) -! nest - do k = 1,nkde - pre_n = mu_m * ngrid%c3f(k) + p_top_m + ngrid%c4f(k) - alt_w_n(k) = -hsca_m * alog(pre_n/p_surf_m) - enddo - do k = 1,nkde-1 - pre_n = mu_m * ngrid%c3h(k) + p_top_m + ngrid%c4h(k) - alt_u_n(k+1) = -hsca_m * alog(pre_n/p_surf_m) - enddo - alt_u_n(1) = alt_w_n(1) - alt_u_n(nkde+1) = alt_w_n(nkde) -endif - - - - !KAL added this call for vertical nesting (return coarse grid dimensions to intended values) - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - - CALL get_ijk_from_grid ( grid , & - ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe ) - - -if (ngrid%vert_refine_method .NE. 0) then - -!KAL added this code (the include file) for the vertical nesting -#include "nest_interpdown_interp_vert.inc" - - - !KAL finish off the 1-D variables (t_base, u_base, v_base, qv_base, and z_base) (move this out of here if alt_u_c and alt_u_n are calculated elsewhere) - CALL vert_interp_vert_nesting_1d ( & - ngrid%t_base, & ! CD field - ids, ide, kds, kde, jds, jde, & ! CD dims - ims, ime, kms, kme, jms, jme, & ! CD dims - ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims - pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid - pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants - alt_u_c, alt_u_n) ! coordinates for parent and nest - CALL vert_interp_vert_nesting_1d ( & - ngrid%u_base, & ! CD field - ids, ide, kds, kde, jds, jde, & ! CD dims - ims, ime, kms, kme, jms, jme, & ! CD dims - ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims - pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid - pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants - alt_u_c, alt_u_n) ! coordinates for parent and nest - CALL vert_interp_vert_nesting_1d ( & - ngrid%v_base, & ! CD field - ids, ide, kds, kde, jds, jde, & ! CD dims - ims, ime, kms, kme, jms, jme, & ! CD dims - ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims - pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid - pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants - alt_u_c, alt_u_n) ! coordinates for parent and nest - CALL vert_interp_vert_nesting_1d ( & - ngrid%qv_base, & ! CD field - ids, ide, kds, kde, jds, jde, & ! CD dims - ims, ime, kms, kme, jms, jme, & ! CD dims - ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims - pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid - pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants - alt_u_c, alt_u_n) ! coordinates for parent and nest - CALL vert_interp_vert_nesting_1d ( & - ngrid%z_base, & ! CD field - ids, ide, kds, kde, jds, jde, & ! CD dims - ims, ime, kms, kme, jms, jme, & ! CD dims - ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims - pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid - pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants - alt_u_c, alt_u_n) ! coordinates for parent and nest - -endif - - CALL push_communicators_for_domain( grid%id ) - -#include "HALO_INTERP_DOWN.inc" - - CALL pop_communicators_for_domain - -# include "nest_interpdown_interp.inc" - - RETURN - END SUBROUTINE interp_domain_em_part2 - -!------------------------------------------------------------------ - - SUBROUTINE interp_domain_em_small_part1 ( grid, intermediate_grid, ngrid, config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type - USE module_comm_dm, ONLY: halo_em_horiz_interp_sub - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & - mytask, get_dm_max_halo_width, & - nest_task_offsets, mpi_comm_to_kid, mpi_comm_to_mom, & - which_kid, nest_pes_x, nest_pes_y, intercomm_active - USE module_timing - IMPLICIT NONE -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: intermediate_grid - TYPE(domain), POINTER :: ngrid -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k - INTEGER iparstrt,jparstrt,sw - TYPE (grid_config_rec_type) :: config_flags - REAL xv(2000) - INTEGER :: ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe - - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: iids, iide, ijds, ijde, ikds, ikde, & - iims, iime, ijms, ijme, ikms, ikme, & - iips, iipe, ijps, ijpe, ikps, ikpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER icoord, jcoord, idim_cd, jdim_cd, pgr - INTEGER thisdomain_max_halo_width - INTEGER local_comm, myproc, nproc - INTEGER ioffset - - CALL wrf_get_dm_communicator ( local_comm ) - CALL wrf_get_myproc( myproc ) - CALL wrf_get_nproc( nproc ) - - CALL get_ijk_from_grid ( grid , & - ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe ) -#ifdef DM_PARALLEL -# include "HALO_EM_HORIZ_INTERP.inc" -#endif - - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - CALL get_ijk_from_grid ( intermediate_grid , & - iids, iide, ijds, ijde, ikds, ikde, & - iims, iime, ijms, ijme, ikms, ikme, & - iips, iipe, ijps, ijpe, ikps, ikpe ) - CALL get_ijk_from_grid ( ngrid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - - CALL nl_get_parent_grid_ratio ( ngrid%id, pgr ) - CALL nl_get_i_parent_start ( intermediate_grid%id, iparstrt ) - CALL nl_get_j_parent_start ( intermediate_grid%id, jparstrt ) - CALL nl_get_shw ( intermediate_grid%id, sw ) - icoord = iparstrt - sw - jcoord = jparstrt - sw - idim_cd = iide - iids + 1 - jdim_cd = ijde - ijds + 1 - - nlev = ckde - ckds + 1 - - ! get max_halo_width for parent. It may be smaller if it is moad - CALL get_dm_max_halo_width ( grid%id , thisdomain_max_halo_width ) - - ! How many 3d arrays, so far just 3d theta-300 and geopotential perturbation, - ! and the 2d topo elevation, three max press/temp/height fields, and three - ! min press/temp/height fields. - - msize = ( 2 )* nlev + 7 - -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, calling rsl_lite_to_child') - CALL rsl_lite_to_child_info( local_communicator, msize*RWORDSIZE & - ,cips,cipe,cjps,cjpe & - ,iids,iide,ijds,ijde & - ,nids,nide,njds,njde & - ,pgr , sw & - ,ntasks_x,ntasks_y & - ,thisdomain_max_halo_width & - ,icoord,jcoord & - ,idim_cd,jdim_cd & - ,pig,pjg,retval ) -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, back from rsl_lite_to_child') - DO while ( retval .eq. 1 ) - IF ( SIZE(grid%ph_2) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ph_2') - DO k = ckds,ckde - xv(k)= grid%ph_2(pig,k,pjg) - END DO - CALL rsl_lite_to_child_msg(((ckde)-(ckds)+1)*RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%t_2) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, t_2') - DO k = ckds,(ckde-1) - xv(k)= grid%t_2(pig,k,pjg) - END DO - CALL rsl_lite_to_child_msg((((ckde-1))-(ckds)+1)*RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%ht) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ht') - xv(1)= grid%ht(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%t_max_p) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, t_max_p') - xv(1)= grid%t_max_p(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%ght_max_p) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ght_max_p') - xv(1)= grid%ght_max_p(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%max_p) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, max_p') - xv(1)= grid%max_p(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%t_min_p) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, t_min_p') - xv(1)= grid%t_min_p(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%ght_min_p) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ght_min_p') - xv(1)= grid%ght_min_p(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%min_p) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, min_p') - xv(1)= grid%min_p(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, calling rsl_lite_to_child_info') - CALL rsl_lite_to_child_info( local_communicator, msize*RWORDSIZE & - ,cips,cipe,cjps,cjpe & - ,iids,iide,ijds,ijde & - ,nids,nide,njds,njde & - ,pgr , sw & - ,ntasks_x,ntasks_y & - ,thisdomain_max_halo_width & - ,icoord,jcoord & - ,idim_cd,jdim_cd & - ,pig,pjg,retval ) -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, back from rsl_lite_to_child_info') - END DO - - ! determine which communicator and offset to use - IF ( intercomm_active( grid%id ) ) THEN ! I am parent - local_comm = mpi_comm_to_kid( which_kid(ngrid%id), grid%id ) - ioffset = nest_task_offsets(ngrid%id) - ELSE IF ( intercomm_active( ngrid%id ) ) THEN ! I am nest - local_comm = mpi_comm_to_mom( ngrid%id ) - ioffset = nest_task_offsets(ngrid%id) - END IF - -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, calling rsl_lite_bcast') - CALL rsl_lite_bcast_msgs( myproc, nest_pes_x(grid%id)*nest_pes_y(grid%id), & - nest_pes_x(ngrid%id)*nest_pes_y(ngrid%id), & - ioffset, local_comm ) -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, back from rsl_lite_bcast') - - RETURN - END SUBROUTINE interp_domain_em_small_part1 - -!------------------------------------------------------------------ - - SUBROUTINE interp_domain_em_small_part2 ( grid, ngrid, config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & - mytask, get_dm_max_halo_width - USE module_comm_nesting_dm, ONLY : halo_interp_down_sub - IMPLICIT NONE -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: ngrid -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k - TYPE (grid_config_rec_type) :: config_flags - REAL xv(2000) - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - INTEGER :: ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER myproc - INTEGER ierr - INTEGER thisdomain_max_halo_width - - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - CALL get_ijk_from_grid ( ngrid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - - nlev = ckde - ckds + 1 - - CALL get_dm_max_halo_width ( ngrid%id , thisdomain_max_halo_width ) - - CALL rsl_lite_from_parent_info(pig,pjg,retval) - - DO while ( retval .eq. 1 ) - - IF ( SIZE(grid%ph_2) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(((ckde)-(ckds)+1)*RWORDSIZE,xv) - DO k = ckds,ckde - grid%ph_2(pig,k,pjg) = xv(k) - END DO - END IF - - IF ( SIZE(grid%t_2) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg((((ckde-1))-(ckds)+1)*RWORDSIZE,xv) - DO k = ckds,(ckde-1) - grid%t_2(pig,k,pjg) = xv(k) - END DO - END IF - - IF ( SIZE(grid%ht) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%ht(pig,pjg) = xv(1) - END IF - - IF ( SIZE(grid%t_max_p) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%t_max_p(pig,pjg) = xv(1) - END IF - - IF ( SIZE(grid%ght_max_p) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%ght_max_p(pig,pjg) = xv(1) - END IF - - IF ( SIZE(grid%max_p) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%max_p(pig,pjg) = xv(1) - END IF - - IF ( SIZE(grid%t_min_p) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%t_min_p(pig,pjg) = xv(1) - END IF - - IF ( SIZE(grid%ght_min_p) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%ght_min_p(pig,pjg) = xv(1) - END IF - - IF ( SIZE(grid%min_p) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%min_p(pig,pjg) = xv(1) - END IF - - CALL rsl_lite_from_parent_info(pig,pjg,retval) - - END DO - - CALL get_ijk_from_grid ( grid , & - ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe ) - -#include "HALO_INTERP_DOWN.inc" - - CALL interp_fcn_bl ( grid%ph_2, & - cids, cide, ckds, ckde, cjds, cjde, & - cims, cime, ckms, ckme, cjms, cjme, & - cips, cipe, ckps, MIN( ckde, ckpe ), cjps, cjpe, & - ngrid%ph_2, & - nids, nide, nkds, nkde, njds, njde, & - nims, nime, nkms, nkme, njms, njme, & - nips, nipe, nkps, MIN( nkde, nkpe ), njps, njpe, & - config_flags%shw, ngrid%imask_nostag, & - .FALSE., .FALSE., & - ngrid%i_parent_start, ngrid%j_parent_start, & - ngrid%parent_grid_ratio, ngrid%parent_grid_ratio, & - grid%ht, ngrid%ht, & - grid%t_max_p, ngrid%t_max_p, & - grid%ght_max_p, ngrid%ght_max_p, & - grid%max_p, ngrid%max_p, & - grid%t_min_p, ngrid%t_min_p, & - grid%ght_min_p, ngrid%ght_min_p, & - grid%min_p, ngrid%min_p, & - ngrid%znw, ngrid%p_top ) - - CALL interp_fcn_bl ( grid%t_2, & - cids, cide, ckds, ckde, cjds, cjde, & - cims, cime, ckms, ckme, cjms, cjme, & - cips, cipe, ckps, MIN( (ckde-1), ckpe ), cjps, cjpe, & - ngrid%t_2, & - nids, nide, nkds, nkde, njds, njde, & - nims, nime, nkms, nkme, njms, njme, & - nips, nipe, nkps, MIN( (nkde-1), nkpe ), njps, njpe, & - config_flags%shw, ngrid%imask_nostag, & - .FALSE., .FALSE., & - ngrid%i_parent_start, ngrid%j_parent_start, & - ngrid%parent_grid_ratio, ngrid%parent_grid_ratio, & - grid%ht, ngrid%ht, & - grid%t_max_p, ngrid%t_max_p, & - grid%ght_max_p, ngrid%ght_max_p, & - grid%max_p, ngrid%max_p, & - grid%t_min_p, ngrid%t_min_p, & - grid%ght_min_p, ngrid%ght_min_p, & - grid%min_p, ngrid%min_p, & - ngrid%znu, ngrid%p_top ) - - RETURN - END SUBROUTINE interp_domain_em_small_part2 - -!------------------------------------------------------------------ - - SUBROUTINE feedback_nest_prep ( grid, config_flags & -! -#include "dummy_new_args.inc" -! -) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, mytask !, & - !push_communicators_for_domain, pop_communicators_for_domain - USE module_comm_nesting_dm, ONLY : halo_interp_up_sub - IMPLICIT NONE -! - TYPE(domain), TARGET :: grid ! name of the grid being dereferenced (must be "grid") - TYPE (grid_config_rec_type) :: config_flags ! configureation flags, has vertical dim of - ! soil temp, moisture, etc., has vertical dim - ! of soil categories -#include "dummy_new_decl.inc" - - INTEGER :: ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER :: idum1, idum2 - - - CALL get_ijk_from_grid ( grid , & - ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe ) - - IF ( grid%active_this_task ) THEN - CALL push_communicators_for_domain( grid%id ) - -#ifdef DM_PARALLEL -#include "HALO_INTERP_UP.inc" -#endif - - CALL pop_communicators_for_domain - END IF - - END SUBROUTINE feedback_nest_prep - -!------------------------------------------------------------------ - - SUBROUTINE feedback_domain_em_part1 ( grid, ngrid, config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type, model_config_rec, model_to_grid_config_rec - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, mytask, & - ipe_save, jpe_save, ips_save, jps_save, & - nest_pes_x, nest_pes_y - - IMPLICIT NONE -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: ngrid -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k - TYPE(domain), POINTER :: xgrid - TYPE (grid_config_rec_type) :: config_flags, nconfig_flags - REAL xv(2000) - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER local_comm, myproc, nproc, idum1, idum2 - INTEGER thisdomain_max_halo_width - -!cyl: add variables for trajectory - integer tjk - - INTERFACE - SUBROUTINE feedback_nest_prep ( grid, config_flags & -! -#include "dummy_new_args.inc" -! -) - USE module_state_description - USE module_domain, ONLY : domain - USE module_configure, ONLY : grid_config_rec_type -! - TYPE (grid_config_rec_type) :: config_flags - TYPE(domain), TARGET :: grid -#include "dummy_new_decl.inc" - END SUBROUTINE feedback_nest_prep - END INTERFACE -! - - CALL wrf_get_dm_communicator ( local_comm ) - CALL wrf_get_myproc( myproc ) - CALL wrf_get_nproc( nproc ) - -! -! intermediate grid - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) -! nest grid - CALL get_ijk_from_grid ( ngrid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - - nlev = ckde - ckds + 1 - - ips_save = ngrid%i_parent_start ! used in feedback_domain_em_part2 below - jps_save = ngrid%j_parent_start - ipe_save = ngrid%i_parent_start + (nide-nids+1) / ngrid%parent_grid_ratio - 1 - jpe_save = ngrid%j_parent_start + (njde-njds+1) / ngrid%parent_grid_ratio - 1 - -! feedback_nest_prep invokes a halo exchange on the ngrid. It is done this way -! in a separate routine because the HALOs need the data to be dereference from the -! grid data structure and, in this routine, the dereferenced fields are related to -! the intermediate domain, not the nest itself. Save the current grid pointer to intermediate -! domain, switch grid to point to ngrid, invoke feedback_nest_prep, then restore grid -! to point to intermediate domain. - - CALL model_to_grid_config_rec ( ngrid%id , model_config_rec , nconfig_flags ) - CALL set_scalar_indices_from_config ( ngrid%id , idum1 , idum2 ) - xgrid => grid - grid => ngrid - - CALL feedback_nest_prep ( grid, nconfig_flags & -! -#include "actual_new_args.inc" -! -) - -! put things back so grid is intermediate grid - - grid => xgrid - CALL set_scalar_indices_from_config ( grid%id , idum1 , idum2 ) - -! "interp" (basically copy) ngrid onto intermediate grid - -#include "nest_feedbackup_interp.inc" - - RETURN - END SUBROUTINE feedback_domain_em_part1 - -!------------------------------------------------------------------ - - SUBROUTINE feedback_domain_em_part2 ( grid, intermediate_grid, ngrid , config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, domain_clock_get, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type, model_config_rec - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, mytask, & - ipe_save, jpe_save, ips_save, jps_save, get_dm_max_halo_width, & - nest_pes_x, nest_pes_y, & - intercomm_active, nest_task_offsets, & - mpi_comm_to_mom, mpi_comm_to_kid, which_kid !, & - !push_communicators_for_domain, pop_communicators_for_domain - - USE module_comm_nesting_dm, ONLY : halo_interp_up_sub - USE module_utility - IMPLICIT NONE - -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: intermediate_grid - TYPE(domain), POINTER :: ngrid - TYPE(domain), POINTER :: parent_grid - -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k - TYPE (grid_config_rec_type) :: config_flags - REAL xv(2000) - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - INTEGER :: xids, xide, xjds, xjde, xkds, xkde, & - xims, xime, xjms, xjme, xkms, xkme, & - xips, xipe, xjps, xjpe, xkps, xkpe - INTEGER :: ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER icoord, jcoord, idim_cd, jdim_cd - INTEGER local_comm, myproc, nproc, ioffset - INTEGER iparstrt, jparstrt, sw, thisdomain_max_halo_width - REAL nest_influence - - character*256 :: timestr - integer ierr - - LOGICAL, EXTERNAL :: cd_feedback_mask - -!cyl: add variables for trajectory - integer tjk - -! On entry to this routine, -! "grid" refers to the parent domain -! "intermediate_grid" refers to local copy of parent domain that overlies this patch of nest -! "ngrid" refers to the nest, which is only needed for smoothing on the parent because -! the nest feedback data has already been transferred during em_nest_feedbackup_interp -! in part1, above. -! The way these settings c and n dimensions are set, below, looks backwards but from the point -! of view of the RSL routine rsl_lite_to_parent_info(), call to which is included by -! em_nest_feedbackup_pack, the "n" domain represents the parent domain and the "c" domain -! represents the intermediate domain. The backwards lookingness should be fixed in the gen_comms.c -! registry routine that accompanies RSL_LITE but, just as it's sometimes easier to put up a road -! sign that says "DIP" than fix the dip, at this point it was easier just to write this comment. JM -! - nest_influence = 1. - - CALL domain_clock_get( grid, current_timestr=timestr ) - - CALL get_ijk_from_grid ( intermediate_grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - CALL get_ijk_from_grid ( grid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - CALL get_ijk_from_grid ( ngrid , & - xids, xide, xjds, xjde, xkds, xkde, & - xims, xime, xjms, xjme, xkms, xkme, & - xips, xipe, xjps, xjpe, xkps, xkpe ) - - ips_save = ngrid%i_parent_start ! used in feedback_domain_em_part2 below - jps_save = ngrid%j_parent_start - ipe_save = ngrid%i_parent_start + (xide-xids+1) / ngrid%parent_grid_ratio - 1 - jpe_save = ngrid%j_parent_start + (xjde-xjds+1) / ngrid%parent_grid_ratio - 1 - - - - -IF ( ngrid%active_this_task ) THEN -!cyl add this for trajectory - CALL push_communicators_for_domain( ngrid%id ) - - do tjk = 1,config_flags%num_traj - if (ngrid%traj_long(tjk) .eq. -9999.0) then -! print*,'n=-9999',tjk - ngrid%traj_long(tjk)=grid%traj_long(tjk) - ngrid%traj_k(tjk)=grid%traj_k(tjk) - else -! print*,'p!=-9999',tjk - grid%traj_long(tjk)=ngrid%traj_long(tjk) - grid%traj_k(tjk)=ngrid%traj_k(tjk) - endif - if (ngrid%traj_lat(tjk) .eq. -9999.0) then - ngrid%traj_lat(tjk)=grid%traj_lat(tjk) - ngrid%traj_k(tjk)=grid%traj_k(tjk) - else - grid%traj_lat(tjk)=ngrid%traj_lat(tjk) - grid%traj_k(tjk)=ngrid%traj_k(tjk) - endif - enddo -!endcyl - - CALL nl_get_i_parent_start ( intermediate_grid%id, iparstrt ) - CALL nl_get_j_parent_start ( intermediate_grid%id, jparstrt ) - CALL nl_get_shw ( intermediate_grid%id, sw ) - icoord = iparstrt - sw - jcoord = jparstrt - sw - idim_cd = cide - cids + 1 - jdim_cd = cjde - cjds + 1 - - nlev = ckde - ckds + 1 - - CALL get_dm_max_halo_width ( grid%id , thisdomain_max_halo_width ) - - parent_grid => grid - grid => ngrid -#include "nest_feedbackup_pack.inc" - grid => parent_grid - CALL pop_communicators_for_domain - -END IF - -! CALL wrf_get_dm_communicator ( local_comm ) -! CALL wrf_get_myproc( myproc ) -! CALL wrf_get_nproc( nproc ) - - ! determine which communicator and offset to use - IF ( intercomm_active( grid%id ) ) THEN ! I am parent - local_comm = mpi_comm_to_kid( which_kid(ngrid%id), grid%id ) - ioffset = nest_task_offsets(ngrid%id) - ELSE IF ( intercomm_active( ngrid%id ) ) THEN ! I am nest - local_comm = mpi_comm_to_mom( ngrid%id ) - ioffset = nest_task_offsets(ngrid%id) - END IF - - IF ( grid%active_this_task .OR. ngrid%active_this_task ) THEN -#ifndef STUBMPI - CALL mpi_comm_rank(local_comm,myproc,ierr) - CALL mpi_comm_size(local_comm,nproc,ierr) -#endif -!call tracebackqq() - CALL rsl_lite_merge_msgs( myproc, nest_pes_x(grid%id)*nest_pes_y(grid%id), & - nest_pes_x(ngrid%id)*nest_pes_y(ngrid%id), & - ioffset, local_comm ) - END IF - -IF ( grid%active_this_task ) THEN - CALL push_communicators_for_domain( grid%id ) - - -#define NEST_INFLUENCE(A,B) A = B -#include "nest_feedbackup_unpack.inc" - - ! smooth coarse grid - CALL get_ijk_from_grid ( ngrid, & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - CALL get_ijk_from_grid ( grid , & - ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe ) - -#include "HALO_INTERP_UP.inc" - - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - -#include "nest_feedbackup_smooth.inc" - - CALL pop_communicators_for_domain -END IF - - RETURN - END SUBROUTINE feedback_domain_em_part2 -#endif - - -!------------------------------------------------------------------ - SUBROUTINE wrf_gatherv_real (Field, field_ofst, & my_count , & ! sendcount globbuf, glob_ofst , & ! recvbuf diff --git a/frame/CMakeLists.txt b/frame/CMakeLists.txt index 2882d036ed..e57efbff60 100644 --- a/frame/CMakeLists.txt +++ b/frame/CMakeLists.txt @@ -87,7 +87,17 @@ target_include_directories( set( MODULE_DM module_dm_stubs.F ) if ( ${USE_RSL_LITE} ) message( STATUS "Setting module_dm to RSL_LITE" ) - set( MODULE_DM ${PROJECT_SOURCE_DIR}/external/RSL_LITE/module_dm.F ) + set( + MODULE_DM + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/module_dm.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/feedback_domain_em_part1.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/feedback_domain_em_part2.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/force_domain_em_part2.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/interp_domain_em_part1.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/interp_domain_em_part2.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/interp_domain_em_part3.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/interp_domain_em_small.F + ) endif() target_sources( diff --git a/frame/Makefile b/frame/Makefile index 75973c87d3..9dea622a54 100644 --- a/frame/Makefile +++ b/frame/Makefile @@ -1,4 +1,5 @@ # +include ../configure.wrf LN = ln -sf MAKE = make -i -r @@ -95,16 +96,25 @@ OBJS = \ hires_timer.o \ clog.o +ifeq ($(DMPARALLEL),1) + RSL_OBJS= \ + ../external/RSL_LITE/feedback_domain_em_part1.o \ + ../external/RSL_LITE/feedback_domain_em_part2.o \ + ../external/RSL_LITE/force_domain_em_part2.o \ + ../external/RSL_LITE/interp_domain_em_part1.o \ + ../external/RSL_LITE/interp_domain_em_part2.o \ + ../external/RSL_LITE/interp_domain_em_part3.o \ + ../external/RSL_LITE/interp_domain_em_small.o +endif + #compile as a .o but do not link into the main library SPECIAL = module_internal_header_util.o pack_utils.o - -include ../configure.wrf LIBTARGET = framework TARGETDIR = ./ -$(LIBTARGET) : $(MODULES) $(OBJS) $(SPECIAL) $(NLOBJS) $(ALOBJS) - $(AR) $(ARFLAGS) ../main/$(LIBWRFLIB) $(MODULES) $(OBJS) $(NLOBJS) $(ALOBJS) +$(LIBTARGET) : $(MODULES) $(OBJS) $(SPECIAL) $(NLOBJS) $(ALOBJS) $(RSL_OBJS) + $(AR) $(ARFLAGS) ../main/$(LIBWRFLIB) $(MODULES) $(OBJS) $(NLOBJS) $(ALOBJS) $(RSL_OBJS) $(RANLIB) ../main/$(LIBWRFLIB) nl_set_0_routines.o : nl_access_routines.F module_configure.o diff --git a/main/depend.common b/main/depend.common index f76684e3b9..05eae4ff9b 100644 --- a/main/depend.common +++ b/main/depend.common @@ -21,6 +21,64 @@ module_dm.o: \ module_cpl.o \ ../share/module_model_constants.o +../external/RSL_LITE/feedback_domain_em_part1.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o + + +../external/RSL_LITE/feedback_domain_em_part2.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o \ + module_comm_nesting_dm.o \ + ../external/esmf_time_f90/module_utility.o + + +../external/RSL_LITE/force_domain_em_part2.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o \ + ../share/module_model_constants.o \ + module_comm_nesting_dm.o + + +../external/RSL_LITE/interp_domain_em_part1.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o \ + module_timing.o + + +../external/RSL_LITE/interp_domain_em_part2.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o \ + module_comm_nesting_dm.o + + +../external/RSL_LITE/interp_domain_em_part3.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o \ + module_comm_nesting_dm.o + + +../external/RSL_LITE/interp_domain_em_small.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o \ + module_comm_dm.o \ + module_comm_nesting_dm.o \ + module_timing.o + module_timing.o: \ module_wrf_error.o \ diff --git a/share/mediation_interp_domain.F b/share/mediation_interp_domain.F index 57abe41cdd..ccd3ec1bad 100644 --- a/share/mediation_interp_domain.F +++ b/share/mediation_interp_domain.F @@ -51,6 +51,21 @@ SUBROUTINE interp_domain_em_part2 ( grid, nested_grid, parent_grid, config_flags TYPE (grid_config_rec_type) :: config_flags # include "dummy_new_decl.inc" END SUBROUTINE interp_domain_em_part2 + + + SUBROUTINE interp_domain_em_part3 ( grid, nested_grid, parent_grid, config_flags & +! +# include "dummy_new_args.inc" +! + ) + USE module_domain + USE module_configure + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: nested_grid + TYPE(domain), POINTER :: parent_grid !KAL added for vertical nesting + TYPE (grid_config_rec_type) :: config_flags +# include "dummy_new_decl.inc" + END SUBROUTINE interp_domain_em_part3 #endif END INTERFACE ! ---------------------------------------------------------- @@ -99,6 +114,11 @@ END SUBROUTINE interp_domain_em_part2 CALL interp_domain_em_part2 ( grid, nested_grid, parent_grid, config_flags & ! # include "actual_new_args.inc" +! + ) + CALL interp_domain_em_part3 ( grid, nested_grid, parent_grid, config_flags & +! +# include "actual_new_args.inc" ! ) ENDIF From 6e71a0aacaaee4e655693822c81f1dd988aa376e Mon Sep 17 00:00:00 2001 From: "Ted Mansell (NOAA/NSSL)" <37668594+MicroTed@users.noreply.github.com> Date: Fri, 11 Oct 2024 18:51:39 -0500 Subject: [PATCH 17/30] Case insensitive F90 compile rule (#2057) The .F90 source files in physics_mmm need to be compiled directly to .o on case-insensitive file systems (like MacOS standard FS). This PR avoids the creation of a .f90 temporary, which will clobber the .F90 version. TYPE: [bug fix] KEYWORDS: case-insensitive filesystem SOURCE: "Ted Mansell (NOAA/NSSL)" DESCRIPTION OF CHANGES: Problem: 1. On case-insensitive file systems (like MacOS standard FS), the creation of .f90 temporary files from .F90 source will clobber the source files. 2. The configure.defaults has an unrecognized option for Apple clang Solution: 1. In postamble: Removed the .F90.f90 rule and compacted the .F90.o rule to use only the fortran compiler to do any necessary preprocessing. It is convention among fortran compilers that .F90 will be preprocessed. 2. In configure.defaults: For Darwin compile with ifort+clang, removed the -qopenmp from OMPCC (Apple clang does not support OpenMP) ISSUE: Addresses .F90.f90 issue raised in https://github.com/wrf-model/WRF/pull/1989 LIST OF MODIFIED FILES: arch/postamble arch/configure.defaults TESTS CONDUCTED: 1. Compiles correctly on MacOS (Intel CPU with ifort + clang) and Linux 2. Are the Jenkins tests all passing? Yes. RELEASE NOTE: Fixed a compile problem with .F90 source files on case-insensitive file systems. --- arch/postamble | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/arch/postamble b/arch/postamble index aa55662073..b5e585d29e 100644 --- a/arch/postamble +++ b/arch/postamble @@ -207,9 +207,9 @@ wrfio_esmf : $(RM) $@ sed -e "s/^\!.*'.*//" -e "s/^ *\!.*'.*//" $*.F90 > $*.G $(CPP) -I$(WRF_SRC_ROOT_DIR)/inc $(CPPFLAGS) $(OMPCPP) $*.G > $*.bb - $(SED_FTN) $*.bb | $(CPP) $(TRADFLAG) > $*.f90 + $(SED_FTN) $*.bb | $(CPP) $(TRADFLAG) > $*.tmp.f90 $(RM) $*.G $*.bb - $(FC) -o $@ -c $(FCFLAGS) $(OMP) $(MODULE_DIRS) $(PROMOTION) $(FCSUFFIX) $*.f90 + $(FC) -o $@ -c $(FCFLAGS) $(OMP) $(MODULE_DIRS) $(PROMOTION) $(FCSUFFIX) $*.tmp.f90 .F.f90: $(RM) $@ @@ -218,13 +218,6 @@ wrfio_esmf : $(CPP) -I$(WRF_SRC_ROOT_DIR)/inc $(CPPFLAGS) $*.H > $@ $(RM) $*.G $*.H -.F90.f90: - $(RM) $@ - sed -e "s/^\!.*'.*//" -e "s/^ *\!.*'.*//" $*.F90 > $*.G - $(SED_FTN) $*.G > $*.H - $(CPP) -I$(WRF_SRC_ROOT_DIR)/inc $(CPPFLAGS) $*.H > $@ - $(RM) $*.G $*.H - .f90.o: $(RM) $@ $(FC) -o $@ -c $(FCFLAGS) $(PROMOTION) $(FCSUFFIX) $*.f90 From b1ec9641b732615828d1e61189162bd768464de1 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Fri, 11 Oct 2024 17:05:57 -0700 Subject: [PATCH 18/30] CMake build no longer uses generator expressions in defines (#2056) TYPE: bug fix KEYWORDS: cmake, compilation SOURCE: internal DESCRIPTION OF CHANGES: Problem: The use of generator expressions in the defines compacts the logic neatly but removes the ability to evaluate these conditionals at configuration time. As such, assumptions must either be made or defines wholly dropped when adding configure-time commands like C preprocessing, both of which are wrong. Solution: Switch the logic to a more verbose `if()`-style that guarantees defines that can be known at configure time are resolved. LIST OF MODIFIED FILES: M CMakeLists.txt M cmake/c_preproc.cmake RELEASE NOTE: CMake build no longer uses generator expressions in defines --- CMakeLists.txt | 246 +++++++++++++++++++++++++++++------------- cmake/c_preproc.cmake | 9 +- 2 files changed, 175 insertions(+), 80 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d1a1297f2f..e719518246 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -619,55 +619,8 @@ add_compile_definitions( DWORDSIZE=${DWORDSIZE} LWORDSIZE=${LWORDSIZE} RWORDSIZE=${RWORDSIZE} - # Only define if set, this is to use #ifdef/#ifndef preprocessors - # in code since cmake cannot handle basically any others :( - # https://gitlab.kitware.com/cmake/cmake/-/issues/17398 - $<$:WRF_CHEM=$> - $<$:BUILD_CHEM=$> - $<$:WRF_CMAQ=$> - $<$,$>:WRF_KPP=$> - $<$:WRF_DFI_RADAR=$> - $<$:WRF_TITAN=$> - $<$:WRF_MARS=$> - $<$:WRF_VENUS=$> - $<$:WRF_HYDRO=$> - - # Because once again we need two defines to control one thing - $<$:WRF_USE_CTSM=$> - $<$>:WRF_USE_CLM> - - # If force classic or no nc-4 support enable classic - $<$,$>>:NETCDF_classic=1> - $<$,$>>:WRFIO_NCD_NO_LARGE_FILE_SUPPORT=1> - # May need a check for WRFIO_ncdpar_LARGE_FILE_SUPPORT - - # Now set the opposite in different defines, because why not :) - $<$>,$>:USE_NETCDF4_FEATURES=1> - $<$>,$>:WRFIO_NCD_LARGE_FILE_SUPPORT=1> - - # Could simplify logic to just check if RPC is available but to be explicit - # Does this actually need to check for EM_CORE (Config.pl:443) - # not enable terran or not rpc_found do - # not ( enable terrain and rpc_found ) - $<$,$>>:LANDREAD_STUB> - $<$:TERRAIN_AND_LANDUSE> - - - $<$:USE_ALLOCATABLES> - $<$:wrfmodel> - $<$:GRIB1> - $<$:INTIO> - $<$:KEEP_INT_AROUND> - $<$:LIMIT_ARGS> - - #!TODO Always defined - fix the ambiguous english in these BUILD_*_FAST defines - BUILD_RRTMG_FAST=$ - BUILD_RRTMK=$ - BUILD_SBM_FAST=$ - SHOW_ALL_VARS_USED=$ - - # Alwasys set - NMM_CORE=$ + + NMM_MAX_DIM=2600 NETCDF @@ -675,33 +628,180 @@ add_compile_definitions( NONSTANDARD_SYSTEM_SUBR EM_CORE=${EM_CORE} - WRFPLUS=$> - DA_CORE=$,$>> - # DFI_RADAR=$ - - # Nesting options - $<$:MOVE_NESTS> - $<$>:VORTEX_CENTER> - - # Configuration checks - $<$>:NO_IEEE_MODULE> - $<$>:NO_ISO_C_SUPPORT> - # If flush fails, check if we can fall back to fflush, and if not no support - $<$>:$,USE_FFLUSH,NO_FLUSH_SUPPORT>> - $<$>:NO_GAMMA_SUPPORT> - - #!TODO Leaving as is in WRF for now but investigate why we don't do this - # https://stackoverflow.com/a/1035713 - # If fseeko64 succeeds, use that, else check if we can fall back to fseeko, and if not just use fseek - $,FSEEKO64_OK,$,FSEEKO_OK,FSEEK_OK>> - - # I don't believe these are used anymore... - # $<$:MPI2_SUPPORT=$> - # $<$:MPI2_THREAD_SUPPORT=$> - ) +# Only define if set, this is to use #ifdef/#ifndef preprocessors +# in code since cmake cannot handle basically any others :( +# https://gitlab.kitware.com/cmake/cmake/-/issues/17398 +if ( ${ENABLE_CHEM} ) + add_compile_definitions( WRF_CHEM=1 ) + if ( ${ENABLE_KPP} ) + add_compile_definitions( WRF_KPP=1 ) + endif() +endif() +if ( ${ENABLE_CHEM} ) + add_compile_definitions( BUILD_CHEM=1 ) +endif() +if ( ${ENABLE_CMAQ} ) + add_compile_definitions( WRF_CMAQ=1 ) +endif() +if ( ${ENABLE_DFI_RADAR} ) + add_compile_definitions( WRF_DFI_RADAR=1 ) +endif() +if ( ${ENABLE_TITAN} ) + add_compile_definitions( WRF_TITAN=1 ) +endif() +if ( ${ENABLE_MARS} ) + add_compile_definitions( WRF_MARS=1 ) +endif() +if ( ${ENABLE_VENUS} ) + add_compile_definitions( WRF_VENUS=1 ) +endif() +if ( ${ENABLE_HYDRO} ) + add_compile_definitions( WRF_HYDRO=1 ) +endif() + +# Because once again we need two defines to control one thing +if ( ${ENABLE_CTSM} ) + add_compile_definitions( WRF_USE_CTSM ) +else() + #!TODO there are some files that rely on this being 1, but that is never set by the legacy make system + add_compile_definitions( WRF_USE_CLM ) +endif() + +# If force classic or no nc-4 support enable classic +if ( ${FORCE_NETCDF_CLASSIC} OR ( NOT ${netCDF_NC4} ) ) + add_compile_definitions( NETCDF_classic=1 ) +endif() +if ( ${WRFIO_NCD_NO_LARGE_FILE_SUPPORT} OR ( NOT ${netCDF_LARGE_FILE_SUPPORT} ) ) + add_compile_definitions( WRFIO_NCD_NO_LARGE_FILE_SUPPORT=1 ) +endif() +# May need a check for WRFIO_ncdpar_LARGE_FILE_SUPPORT + +# Now set the opposite in different defines, because why not :) +if ( ( NOT ${FORCE_NETCDF_CLASSIC} ) AND ${netCDF_NC4} ) + add_compile_definitions( USE_NETCDF4_FEATURES=1 ) +endif() +if ( ( NOT ${WRFIO_NCD_NO_LARGE_FILE_SUPPORT} ) AND ${netCDF_LARGE_FILE_SUPPORT} ) + add_compile_definitions( WRFIO_NCD_LARGE_FILE_SUPPORT=1 ) +endif() + +# Could simplify logic to just check if RPC is available but to be explicit +# Does this actually need to check for EM_CORE (Config.pl:443) +# not enable terran or not rpc_found do +# not ( enable terrain and rpc_found ) +# if ( NOT ( ${ENABLE_TERRAIN} AND ${RPC_FOUND} ) ) # this is wrong, needs fixing +# add_compile_definitions( LANDREAD_STUB ) +# endif() +if ( ${ENABLE_TERRAIN} AND ${MOVE_NESTS} ) + add_compile_definitions( TERRAIN_AND_LANDUSE ) +else () + add_compile_definitions( LANDREAD_STUB ) +endif() + +if ( ${USE_ALLOCATABLES} ) + add_compile_definitions( USE_ALLOCATABLES ) +endif() +if ( ${wrfmodel} ) + add_compile_definitions( wrfmodel ) +endif() +if ( ${GRIB1} ) + add_compile_definitions( GRIB1 ) +endif() +if ( ${INTIO} ) + add_compile_definitions( INTIO ) +endif() +if ( ${KEEP_INT_AROUND} ) + add_compile_definitions( KEEP_INT_AROUND ) +endif() +if ( ${LIMIT_ARGS} ) + add_compile_definitions( LIMIT_ARGS ) +endif() + + +if ( ${BUILD_RRTMG_FAST} ) + add_compile_definitions( BUILD_RRTMG_FAST=1 ) +else() + add_compile_definitions( BUILD_RRTMG_FAST=0 ) +endif() +if ( ${BUILD_RRTMK} ) + add_compile_definitions( BUILD_RRTMK=1 ) +else() + add_compile_definitions( BUILD_RRTMK=0 ) +endif() +if ( ${BUILD_SBM_FAST} ) + add_compile_definitions( BUILD_SBM_FAST=1 ) +else() + add_compile_definitions( BUILD_SBM_FAST=0 ) +endif() +if ( ${SHOW_ALL_VARS_USED} ) + add_compile_definitions( SHOW_ALL_VARS_USED=1 ) +else() + add_compile_definitions( SHOW_ALL_VARS_USED=0 ) +endif() +if ( ${NMM_CORE} ) + add_compile_definitions( NMM_CORE=1 ) +else() + add_compile_definitions( NMM_CORE=0 ) +endif() + +if ( "${WRF_CORE}" STREQUAL "PLUS" ) + add_compile_definitions( WRFPLUS=1 ) +else() + add_compile_definitions( WRFPLUS=0 ) +endif() + +if ( "${WRF_CORE}" STREQUAL "DA_CORE" OR "${WRF_CORE}" STREQUAL "DA_4D_VAR" ) + add_compile_definitions( DA_CORE=1 ) +else() + add_compile_definitions( DA_CORE=0 ) +endif() +# DFI_RADAR=$ + +# Nesting options +if ( ${MOVE_NESTS} ) + add_compile_definitions( MOVE_NESTS ) +endif() +if ( "${WRF_NESTING}" STREQUAL "VORTEX" ) + add_compile_definitions( VORTEX_CENTER ) +endif() + +# Configuration checks +if ( NOT ${Fortran_2003_IEEE} ) + add_compile_definitions( NO_IEEE_MODULE ) +endif() +if ( NOT ${Fortran_2003_ISO_C} ) + add_compile_definitions( NO_ISO_C_SUPPORT ) +endif() +# If flush fails, check if we can fall back to fflush, and if not no support +if ( NOT ${Fortran_2003_FLUSH} ) + if ( "${Fortran_2003_FFLUSH}" ) + add_compile_definitions( USE_FFLUSH ) + else() + add_compile_definitions( NO_FLUSH_SUPPORT ) + endif() +endif() +if ( NOT ${Fortran_2003_GAMMA} ) + add_compile_definitions( NO_GAMMA_SUPPORT ) +endif() + +#!TODO Leaving as is in WRF for now but investigate why we don't do this +# https://stackoverflow.com/a/1035713 +# If fseeko64 succeeds, use that, else check if we can fall back to fseeko, and if not just use fseek +if ( NOT ${FSEEKO64} ) + add_compile_definitions( FSEEKO64_OK ) +elseif( "${FSEEKO}" ) + add_compile_definitions( FSEEKO_OK ) +else() + add_compile_definitions( FSEEK_OK ) +endif() + +# I don't believe these are used anymore... +# $<$:MPI2_SUPPORT=$> +# $<$:MPI2_THREAD_SUPPORT=$> + + # Make core target add_library( ${PROJECT_NAME}_Core diff --git a/cmake/c_preproc.cmake b/cmake/c_preproc.cmake index 14f7fe9295..4de0a1e7a1 100644 --- a/cmake/c_preproc.cmake +++ b/cmake/c_preproc.cmake @@ -111,14 +111,9 @@ macro( wrf_expand_definitions ) set( WRF_EXP_DEFS ) foreach( WRF_EXP_DEF ${WRF_EXP_DEFINITIONS} ) if ( NOT ${WRF_EXP_DEF} MATCHES ".*-D.*" ) - # We have a generator expression, inject the -D correctly - # THIS SHOULD ONLY BE USED FOR CONDITIONALLY APPLIED DEFINITIONS + # We have a generator expression, error! no way we can evaluate this correctly if ( ${WRF_EXP_DEF} MATCHES "^[$]<" ) - # Take advantage of the fact that a define is most likely not an expanded variable (i.e. starts with a-zA-Z, adjust if not) - # preceeded by the defining generator expression syntax $<>:var or ,var - # Yes this is fragile but is probably more robust than the current code if you're relying on this macro :D - string( REGEX REPLACE "(>:|,)([a-zA-Z])" "\\1-D\\2" WRF_EXP_DEF_SANITIZED ${WRF_EXP_DEF} ) - list( APPEND WRF_EXP_DEFS ${WRF_EXP_DEF_SANITIZED} ) + message( FATAL_ERROR "Generator expressions not allowed in preprocessing defines" ) else() list( APPEND WRF_EXP_DEFS -D${WRF_EXP_DEF} ) endif() From a9c23379fe29d7a5e08452763183b5eb3154ad5b Mon Sep 17 00:00:00 2001 From: JeroBnd <135150267+JeroBnd@users.noreply.github.com> Date: Mon, 14 Oct 2024 12:20:25 -0300 Subject: [PATCH 19/30] Corrected end date for em_b_wave test case (#2107) TYPE: no impact directly in the run KEYWORDS: namelist, end_date, b_wave SOURCE: Jeronimo Bande (IDING SAS) DESCRIPTION OF CHANGES: Problem: Wrong end date in namelist. Not good for example. Solution: Correct end date of the simulation. LIST OF MODIFIED FILES: test/em_b_wave/namelist.input TESTS CONDUCTED: Jenkins tests are all passing. --- test/em_b_wave/namelist.input | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/em_b_wave/namelist.input b/test/em_b_wave/namelist.input index 0283891fc5..f07ba0dba0 100644 --- a/test/em_b_wave/namelist.input +++ b/test/em_b_wave/namelist.input @@ -11,7 +11,7 @@ start_second = 00, 00, 00, end_year = 0001, 0001, 0001, end_month = 01, 01, 01, - end_day = 05, 05, 05, + end_day = 06, 06, 06, end_hour = 00, 00, 00, end_minute = 00, 00, 00, end_second = 00, 00, 00, From 5ffa840b98f523a020de5a2377d7b2893188007f Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:31:30 -0700 Subject: [PATCH 20/30] Breakout deallocation calls into simpler smaller files (#2070) TYPE: enhancement KEYWORDS: intel, compilation, llvm, memory SOURCE: internal DESCRIPTION OF CHANGES: Problem: The Intel oneAPI compilers (and others like nvhpc) struggle with some of the larger (15k+ lines of code) files within WRF. This causes intense memory usage that is not often available to the average user not in a resource-rich environment. This often limits compilation to single threaded if even possible or to a dedicated environment with enough memory if available. If neither of those is available to a user, they will be unable to use these configurations entirely. Solution: This PR focuses on the `deallocs.inc` sections of code used in `module_domain` to reduce the include size to manageable levels. The include is instead broken out into many smaller files as external subroutines. The files are fully generated source code from the registry, with the calls to the subroutines also being generated as well. This also makes it relatively easy to change the number of files generated from a source code perspective. Build rules would need to be modified accordingly as seen in these changes. TESTS CONDUCTED: Attached to this PR are plots of the respective effects of theses changes. Changes were tested with intel and gcc compilers, but only intel memory usage is shown as it exacerbates the memory usage issue. --- clean | 2 +- frame/CMakeLists.txt | 4 +- frame/Makefile | 20 +++++++-- frame/module_domain.F | 4 -- inc/.gitignore | 2 + main/depend.common | 88 ++++++++++++++++++++++++++++++++++++++ tools/CMakeLists.txt | 14 +++++- tools/gen_allocs.c | 74 ++++++++++++++++++++++++++++++-- tools/protos.h | 2 +- var/build/da.make | 12 ++++++ var/build/da_name_space.pl | 1 + var/build/depend.txt | 37 ++++++++++++++++ 12 files changed, 243 insertions(+), 17 deletions(-) diff --git a/clean b/clean index 85079ddc44..a4af6a3a21 100755 --- a/clean +++ b/clean @@ -21,7 +21,7 @@ endif ( cd tools/CodeBase ; make clean ) -( cd inc ; /bin/rm -f *.inc namelist.default commit_decl *.o *.F ) +( cd inc ; /bin/rm -f *.inc namelist.default commit_decl *.o *.F *.f90 ) find . -name \*.dSYM -exec rm -rf {} \; >& /dev/null diff --git a/frame/CMakeLists.txt b/frame/CMakeLists.txt index e57efbff60..507678afef 100644 --- a/frame/CMakeLists.txt +++ b/frame/CMakeLists.txt @@ -119,8 +119,6 @@ target_sources( module_cpl_oasis3.F - - ${CMAKE_BINARY_DIR}/frame/module_state_description.F # GENERATED ${nl_dyn_source} # GENERATED clog.c @@ -166,4 +164,4 @@ set_source_files_properties( install( FILES ${WRF_INCLUDE_FILES} DESTINATION include/${FOLDER_COMPILE_TARGET} - ) \ No newline at end of file + ) diff --git a/frame/Makefile b/frame/Makefile index 9dea622a54..57d6f98323 100644 --- a/frame/Makefile +++ b/frame/Makefile @@ -85,7 +85,21 @@ NLOBJS =\ nl_set_4_routines.o \ nl_set_5_routines.o \ nl_set_6_routines.o \ - nl_set_7_routines.o + nl_set_7_routines.o + +DEALOBJS =\ + ../inc/deallocs_0.o \ + ../inc/deallocs_1.o \ + ../inc/deallocs_2.o \ + ../inc/deallocs_3.o \ + ../inc/deallocs_4.o \ + ../inc/deallocs_5.o \ + ../inc/deallocs_6.o \ + ../inc/deallocs_7.o \ + ../inc/deallocs_8.o \ + ../inc/deallocs_9.o \ + ../inc/deallocs_10.o \ + ../inc/deallocs_11.o OBJS = \ wrf_num_bytes_between.o \ @@ -113,8 +127,8 @@ SPECIAL = module_internal_header_util.o pack_utils.o LIBTARGET = framework TARGETDIR = ./ -$(LIBTARGET) : $(MODULES) $(OBJS) $(SPECIAL) $(NLOBJS) $(ALOBJS) $(RSL_OBJS) - $(AR) $(ARFLAGS) ../main/$(LIBWRFLIB) $(MODULES) $(OBJS) $(NLOBJS) $(ALOBJS) $(RSL_OBJS) +$(LIBTARGET) : $(MODULES) $(OBJS) $(SPECIAL) $(NLOBJS) $(ALOBJS) $(DEALOBJS) $(RSL_OBJS) + $(AR) $(ARFLAGS) ../main/$(LIBWRFLIB) $(MODULES) $(OBJS) $(NLOBJS) $(ALOBJS) $(DEALOBJS) $(RSL_OBJS) $(RANLIB) ../main/$(LIBWRFLIB) nl_set_0_routines.o : nl_access_routines.F module_configure.o diff --git a/frame/module_domain.F b/frame/module_domain.F index 3a1d3c9bfb..97091e043c 100644 --- a/frame/module_domain.F +++ b/frame/module_domain.F @@ -1646,10 +1646,6 @@ SUBROUTINE dealloc_space_field ( grid ) TYPE(domain) , POINTER :: grid - ! Local data. - - INTEGER :: ierr - # include "deallocs.inc" END SUBROUTINE dealloc_space_field diff --git a/inc/.gitignore b/inc/.gitignore index 42501b8792..2cd4f75d69 100644 --- a/inc/.gitignore +++ b/inc/.gitignore @@ -13,3 +13,5 @@ ############################################################################## *.inc commit_decl +allocs_*.F +deallocs_*.F diff --git a/main/depend.common b/main/depend.common index 05eae4ff9b..9ca1327f5f 100644 --- a/main/depend.common +++ b/main/depend.common @@ -136,6 +136,82 @@ module_comm_nesting_dm.o: \ module_dm_stubs.F: \ module_domain.o +# For the following dealloc_* source files, as they are generated by the registry +# we must make that a dependency. There is no actual registry target and is +# instead facilitated via the module_state_description.F recipe. Placing a dependency on +# module_state_description.F forces the registry to be run if the following source +# files are required implicitly by their respective .o recipe +../inc/deallocs_0.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_0.F : module_state_description.F + + +../inc/deallocs_1.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_1.F : module_state_description.F + + +../inc/deallocs_2.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_2.F : module_state_description.F + + +../inc/deallocs_3.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_3.F : module_state_description.F + + +../inc/deallocs_4.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_4.F : module_state_description.F + + +../inc/deallocs_5.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_5.F : module_state_description.F + + +../inc/deallocs_6.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_6.F : module_state_description.F + + +../inc/deallocs_7.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_7.F : module_state_description.F + + +../inc/deallocs_8.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_8.F : module_state_description.F + + +../inc/deallocs_9.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_9.F : module_state_description.F + + +../inc/deallocs_10.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_10.F : module_state_description.F + + +../inc/deallocs_11.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_11.F : module_state_description.F + module_domain.o: \ module_domain_type.o \ @@ -171,6 +247,18 @@ module_domain.o: \ ../inc/allocs_29.o \ ../inc/allocs_30.o \ ../inc/allocs_31.o \ + ../inc/deallocs_0.o \ + ../inc/deallocs_1.o \ + ../inc/deallocs_2.o \ + ../inc/deallocs_3.o \ + ../inc/deallocs_4.o \ + ../inc/deallocs_5.o \ + ../inc/deallocs_6.o \ + ../inc/deallocs_7.o \ + ../inc/deallocs_8.o \ + ../inc/deallocs_9.o \ + ../inc/deallocs_10.o \ + ../inc/deallocs_11.o \ module_driver_constants.o \ module_configure.o \ module_machine.o \ diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 1ac31ae7b3..57bff48c2c 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -99,6 +99,15 @@ elseif ( ${WRF_CORE} STREQUAL "DA" OR ${WRF_CORE} STREQUAL "DA_4D_VAR" ) endif() +set( dealloc_source ) +foreach( n RANGE 0 11 ) + list( + APPEND + dealloc_source + ${CMAKE_BINARY_DIR}/inc/deallocs_${n}.F + ) +endforeach() + set( allocs_source ) foreach( n RANGE 0 31 ) list( @@ -106,7 +115,6 @@ foreach( n RANGE 0 31 ) allocs_source ${CMAKE_BINARY_DIR}/inc/allocs_${n}.F ) - endforeach() get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) @@ -122,6 +130,7 @@ add_custom_command( OUTPUT ${CMAKE_BINARY_DIR}/inc/nl_config.inc ${CMAKE_BINARY_DIR}/frame/module_state_description.F + ${dealloc_source} ${allocs_source} COMMENT "Generating registry code..." @@ -151,12 +160,15 @@ add_custom_target( DEPENDS ${CMAKE_BINARY_DIR}/inc/nl_config.inc ${CMAKE_BINARY_DIR}/frame/module_state_description.F + ${dealloc_source} ${allocs_source} ) target_sources( ${PROJECT_NAME}_Core PRIVATE + ${CMAKE_BINARY_DIR}/frame/module_state_description.F + ${dealloc_source} ${allocs_source} ) diff --git a/tools/gen_allocs.c b/tools/gen_allocs.c index cff415550f..965a1fc04a 100644 --- a/tools/gen_allocs.c +++ b/tools/gen_allocs.c @@ -257,7 +257,6 @@ gen_alloc1 ( char * dirname ) { stop = start + ( primaryFields / numFiles ); } - printf( "Total [%d] : Setting [start,stop] to [%d, %d]", primaryFields, start, stop ); gen_alloc2( fpSub , "grid%", NULL, &Domain, start, stop, 1 ) ; fprintf( fpSub, @@ -821,30 +820,97 @@ gen_dealloc1 ( char * dirname ) FILE * fp ; char fname[NAMELEN] ; char * fn = "deallocs.inc" ; + // Open array of deallocs_[n].inc + int numFiles = 12; + int idx = 0; + FILE * fpSub; + char * filename_prefix = "deallocs_" ; if ( dirname == NULL ) return(1) ; if ( strlen(dirname) > 0 ) { sprintf(fname,"%s/%s",dirname,fn) ; } else { sprintf(fname,"%s",fn) ; } if ((fp = fopen( fname , "w" )) == NULL ) return(1) ; print_warning(fp,fname) ; - gen_dealloc2( fp , "grid%", &Domain ) ; + + fprintf( + fp, + "INTERFACE\n" + ); + if ( dirname == NULL ) return(1) ; + for ( idx = 0; idx < numFiles; idx++ ) + { + if ( strlen(dirname) > 0 ) { sprintf(fname,"%s/%s%d.F",dirname,filename_prefix,idx) ; } + else { sprintf(fname,"%s%d.F",dirname,filename_prefix,idx ) ; } + if ((fpSub = fopen( fname , "w" )) == NULL ) return(1) ; + + print_warning(fpSub,fname) ; + + fprintf( + fp, + " SUBROUTINE %s%d( grid )\n" + " USE module_wrf_error\n" + " USE module_domain_type\n" + " IMPLICIT NONE\n" + " TYPE( domain ), POINTER :: grid\n END SUBROUTINE\n", + filename_prefix, idx + ); + + fprintf( + fpSub, + "SUBROUTINE %s%d( grid )\n" + " USE module_wrf_error\n" + " USE module_domain_type\n" + " IMPLICIT NONE\n" + " TYPE( domain ), POINTER :: grid\n INTEGER :: ierr\n", + filename_prefix, idx + ); + gen_dealloc2( fpSub, "grid%", &Domain, idx, numFiles ); + fprintf( + fpSub, + "END SUBROUTINE %s%d\n", + filename_prefix, idx + ); + close_the_file( fpSub ) ; + } + fprintf( + fp, + "END INTERFACE\n" + ); + + // Call the functions in the inc + for ( idx = 0; idx < numFiles; idx++ ) + { + fprintf( + fp, + "CALL %s%d( grid )\n", filename_prefix, idx + ); + } close_the_file( fp ) ; return(0) ; } int -gen_dealloc2 ( FILE * fp , char * structname , node_t * node ) +gen_dealloc2 ( FILE * fp , char * structname , node_t * node, int idx, int numFiles ) { node_t * p ; int tag ; char post[NAMELEN] ; char fname[NAMELEN] ; char x[NAMELEN] ; + int currentIdx = -1; if ( node == NULL ) return(1) ; for ( p = node->fields ; p != NULL ; p = p->next ) { + // Modulo to divert each field based on index to a file + // Skip if this field is not part of that index and idx != -1, so -1 can be used to force output + currentIdx = ( currentIdx + 1 ) % numFiles; + if ( currentIdx != idx && idx != -1 ) + { + continue; + } + if ( (p->ndims > 0 || p->boundary_array) && ( /* any array or a boundary array and... */ (p->node_kind & FIELD) || /* scalar arrays or */ (p->node_kind & FOURD) ) /* scalar arrays or */ @@ -914,7 +980,7 @@ structname, fname, structname, fname ) ; else if ( p->type->type_type == DERIVED ) { sprintf(x,"%s%s%%",structname,p->name ) ; - gen_dealloc2(fp,x, p->type) ; + gen_dealloc2(fp,x, p->type, idx, -1) ; } } } diff --git a/tools/protos.h b/tools/protos.h index 6e9fa2dd98..04811e1418 100644 --- a/tools/protos.h +++ b/tools/protos.h @@ -126,7 +126,7 @@ int gen_ddt_write ( char * ); int gen_ddt_write1 ( FILE *, char *, node_t *); int gen_dealloc ( char * ); int gen_dealloc1 ( char * ); -int gen_dealloc2 ( FILE *, char *, node_t *); +int gen_dealloc2 ( FILE *, char *, node_t *, int idx, int numFiles ); int gen_scalar_tables ( FILE *); int AppendReg ( char *,int); int irr_diag_scalar_indices ( char * ); diff --git a/var/build/da.make b/var/build/da.make index 8ed639627c..898d57464c 100644 --- a/var/build/da.make +++ b/var/build/da.make @@ -157,6 +157,18 @@ WRFVAR_OBJS = \ inc/allocs_29.o \ inc/allocs_30.o \ inc/allocs_31.o \ + inc/deallocs_0.o \ + inc/deallocs_1.o \ + inc/deallocs_2.o \ + inc/deallocs_3.o \ + inc/deallocs_4.o \ + inc/deallocs_5.o \ + inc/deallocs_6.o \ + inc/deallocs_7.o \ + inc/deallocs_8.o \ + inc/deallocs_9.o \ + inc/deallocs_10.o \ + inc/deallocs_11.o \ nl_get_0_routines.o \ nl_get_1_routines.o \ nl_get_2_routines.o \ diff --git a/var/build/da_name_space.pl b/var/build/da_name_space.pl index ba01fcd738..b1864b0c1f 100755 --- a/var/build/da_name_space.pl +++ b/var/build/da_name_space.pl @@ -53,6 +53,7 @@ med_auxinput_in med_restart_out med_latbound_in +deallocs_ allocs_ module_configure module_scalar_tables diff --git a/var/build/depend.txt b/var/build/depend.txt index 604d8ed045..3632d9e20e 100644 --- a/var/build/depend.txt +++ b/var/build/depend.txt @@ -12,6 +12,31 @@ mediation_interp_domain.o : mediation_interp_domain.F module_timing.o module_con mediation_nest_move.o : mediation_nest_move.F module_streams.o module_compute_geop.o module_driver_constants.o module_dm.o module_state_description.o module_configure.o module_timing.o module_utility.o module_domain.o mediation_wrfmain.o : mediation_wrfmain.F module_utility.o module_bc_time_utilities.o module_io.o module_timing.o module_io_domain.o module_configure.o module_domain.o module_MPP.o : module_MPP.F +# See main/depend.common for explanation of source file dependency recipes for deallocs +inc/deallocs_0.o : module_wrf_error.o module_domain_type.o +inc/deallocs_0.F : module_state_description.F +inc/deallocs_1.o : module_wrf_error.o module_domain_type.o +inc/deallocs_1.F : module_state_description.F +inc/deallocs_2.o : module_wrf_error.o module_domain_type.o +inc/deallocs_2.F : module_state_description.F +inc/deallocs_3.o : module_wrf_error.o module_domain_type.o +inc/deallocs_3.F : module_state_description.F +inc/deallocs_4.o : module_wrf_error.o module_domain_type.o +inc/deallocs_4.F : module_state_description.F +inc/deallocs_5.o : module_wrf_error.o module_domain_type.o +inc/deallocs_5.F : module_state_description.F +inc/deallocs_6.o : module_wrf_error.o module_domain_type.o +inc/deallocs_6.F : module_state_description.F +inc/deallocs_7.o : module_wrf_error.o module_domain_type.o +inc/deallocs_7.F : module_state_description.F +inc/deallocs_8.o : module_wrf_error.o module_domain_type.o +inc/deallocs_8.F : module_state_description.F +inc/deallocs_9.o : module_wrf_error.o module_domain_type.o +inc/deallocs_9.F : module_state_description.F +inc/deallocs_10.o : module_wrf_error.o module_domain_type.o +inc/deallocs_10.F : module_state_description.F +inc/deallocs_11.o : module_wrf_error.o module_domain_type.o +inc/deallocs_11.F : module_state_description.F module_bc.o : module_bc.F module_state_description.o module_wrf_error.o module_configure.o module_model_constants.o module_bc_time_utilities.o : module_bc_time_utilities.F module_utility.o module_check_a_mundo.o : module_check_a_mundo.F module_configure.o module_wrf_error.o module_state_description.o @@ -59,6 +84,18 @@ module_domain.o : module_domain.F \ inc/allocs_29.o \ inc/allocs_30.o \ inc/allocs_31.o \ + inc/deallocs_0.o \ + inc/deallocs_1.o \ + inc/deallocs_2.o \ + inc/deallocs_3.o \ + inc/deallocs_4.o \ + inc/deallocs_5.o \ + inc/deallocs_6.o \ + inc/deallocs_7.o \ + inc/deallocs_8.o \ + inc/deallocs_9.o \ + inc/deallocs_10.o \ + inc/deallocs_11.o \ module_domain_type.o module_utility.o module_wrf_error.o module_configure.o module_machine.o module_driver_constants.o module_domain_type.o : module_domain_type.F module_streams.o module_utility.o module_driver_constants.o # See main/depend.common for explanation of source file dependency recipes for allocs From cd8e5453fce6942fb24170b2c65bd9d70b432e7a Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Mon, 14 Oct 2024 13:40:49 -0700 Subject: [PATCH 21/30] Change internal flag organization in CMake build to not be global (#2053) TYPE: enhancement KEYWORDS: cmake, flags, compilation SOURCE: internal DESCRIPTION OF CHANGES: Problem: The current iteration of the cmake build places all configuration flags in the global properties of the project. While this works when just building WRF, integration with other projects' cmake builds if placed under WRF pollutes their respective build flags. Solution: Adjust the layout of the flags to instead carry them in a variable, and prefer using `target_*` calls for flag usage. LIST OF MODIFIED FILES: M CMakeLists.txt M chem/CMakeLists.txt M external/CMakeLists.txt M external/io_adios2/CMakeLists.txt M external/io_netcdf/CMakeLists.txt M external/io_netcdfpar/CMakeLists.txt M external/io_pnetcdf/CMakeLists.txt M frame/CMakeLists.txt M main/CMakeLists.txt M phys/CMakeLists.txt M tools/CMakeLists.txt RELEASE NOTE: Change internal flag organization in CMake build to not be global --- CMakeLists.txt | 289 ++++++++++++++------------- chem/CMakeLists.txt | 15 +- external/CMakeLists.txt | 4 + external/io_adios2/CMakeLists.txt | 3 +- external/io_netcdf/CMakeLists.txt | 3 +- external/io_netcdfpar/CMakeLists.txt | 3 +- external/io_pnetcdf/CMakeLists.txt | 3 +- frame/CMakeLists.txt | 5 +- hydro/CMakeLists.txt | 2 + main/CMakeLists.txt | 3 + phys/CMakeLists.txt | 3 +- tools/CMakeLists.txt | 5 +- 12 files changed, 185 insertions(+), 153 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e719518246..0951267f9b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -322,21 +322,21 @@ if ( ${USE_MPI} ) # It still technically finds MPI but the output is nonintuitive # saying things like hdf5 or pthread find_package( MPI REQUIRED COMPONENTS Fortran C ) - add_compile_definitions( - USE_MPI=1 - DM_PARALLEL - ) + list( APPEND PROJECT_COMPILE_DEFINITIONS + USE_MPI=1 + DM_PARALLEL + ) if ( DEFINED WRF_MPI_Fortran_FLAGS AND NOT "${WRF_MPI_Fortran_FLAGS}" STREQUAL "" ) - add_compile_options( - $<$:${WRF_MPI_Fortran_FLAGS}> - ) + list( APPEND PROJECT_COMPILE_OPTIONS + $<$:${WRF_MPI_Fortran_FLAGS}> + ) endif() if ( DEFINED WRF_MPI_C_FLAGS AND NOT "${WRF_MPI_C_FLAGS}" STREQUAL "" ) - add_compile_options( - $<$:${WRF_MPI_C_FLAGS}> - ) + list( APPEND PROJECT_COMPILE_OPTIONS + $<$:${WRF_MPI_C_FLAGS}> + ) endif() # Check if MPI in all its glory has forced IPO down our throats due to hard-coding the wrapper flags @@ -365,16 +365,19 @@ if ( ${USE_MPI} ) set( USE_RSL_LITE ON ) # We know NONE is the zero index so compare against that elseif( ${CURRENT_NESTING_IDX} GREATER 0 ) - add_compile_definitions( - DM_PARALLEL - STUBMPI - ) + list( APPEND PROJECT_COMPILE_DEFINITIONS + DM_PARALLEL + STUBMPI + ) set( USE_RSL_LITE ON ) endif() if ( ${USE_OPENMP} ) find_package( OpenMP REQUIRED COMPONENTS Fortran C ) - add_compile_definitions( USE_OPENMP=1 SM_PARALLEL ) + list( APPEND PROJECT_COMPILE_DEFINITIONS + USE_OPENMP=1 + SM_PARALLEL + ) endif() if ( ${USE_M4} ) @@ -541,50 +544,50 @@ endif() # This is really ugly but such is the cost of supporting many ways to say the same thing # https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_COMPILER_ID.html -add_compile_options( - # Use "" and ; specifically to evaluate correctly - # "$<$:>" #@ Absoft Fortran - # "$<$:>" #@ Analog VisualDSP++ - # "$<$:>" #@ Apple Clang - # "$<$:>" #@ ARM Compiler - # "$<$:>" #@ ARM Compiler based on Clang - # "$<$:>" #@ Bruce C Compiler - # "$<$:>" #@ Concurrent Fortran - # "$<$:>" #@ LLVM Clang - "$<$:-s;integer32;-s;real${RWORDSIZE_B}>" #@ Cray Compiler - # "$<$:>" #@ Embarcadero - "$<$,$>:-fdefault-real-${RWORDSIZE}>" #@ Classic Flang Fortran Compiler - # "$<$:>" #@ LLVM Flang Fortran Compiler - "$<$:-CcdRR${RWORDSIZE}>" #@ Fujitsu HPC compiler (Trad mode) - # "$<$:>" #@ Fujitsu HPC compiler (Clang mode) - "$<$:-r${RWORDSIZE};-i4>" #@ G95 Fortran - "$<$,$>:-fdefault-real-${RWORDSIZE}>" #@ GNU Compiler Collection - # "$<$:>" #@ Green Hills Software - # "$<$:>" #@ Hewlett-Packard Compiler - # "$<$:>" #@ IAR Systems - "$<$:-real-size;${RWORDSIZE_B};-i4>" #@ Intel Classic Compiler - "$<$:-real-size;${RWORDSIZE_B};-i4>" #@ Intel LLVM-Based Compiler - # "$<$:>" #@ MCST Elbrus C/C++/Fortran Compiler - # "$<$:>" #@ Microsoft Visual Studio - "$<$:-r${RWORDSIZE};-i4>" #@ NVIDIA HPC Compiler - # "$<$:>" #@ NVIDIA CUDA Compiler - # "$<$:>" #@ Open Watcom - "$<$:-r${RWORDSIZE};-i4>" #@ The Portland Group - "$<$:-r${RWORDSIZE};-i4>" #@ PathScale - # "$<$:>" #@ Small Device C Compiler - # "$<$:>" #@ Oracle Solaris Studio - # "$<$:>" #@ Tasking Compiler Toolsets - # "$<$:>" #@ Texas Instruments - # "$<$:>" #@ Tiny C Compiler - "$<$:-qrealsize=${RWORDSIZE};-qintsize=4>" #@ IBM XL - # "$<$:>" #@ IBM Clang-based XL - # "$<$:>" #@ IBM LLVM-based Compiler - # Todo find how to handle default selection or add new compiler IDs - # unknown how to add support for sxf90 - - # line lengths - "$<$:-ffree-line-length-none>" #@ GNU Compiler Collection - ) +list( APPEND PROJECT_COMPILE_OPTIONS + # Use "" and ; specifically to evaluate correctly + # "$<$:>" #@ Absoft Fortran + # "$<$:>" #@ Analog VisualDSP++ + # "$<$:>" #@ Apple Clang + # "$<$:>" #@ ARM Compiler + # "$<$:>" #@ ARM Compiler based on Clang + # "$<$:>" #@ Bruce C Compiler + # "$<$:>" #@ Concurrent Fortran + # "$<$:>" #@ LLVM Clang + "$<$:-s;integer32;-s;real${RWORDSIZE_B}>" #@ Cray Compiler + # "$<$:>" #@ Embarcadero + "$<$,$>:-fdefault-real-${RWORDSIZE}>" #@ Classic Flang Fortran Compiler + # "$<$:>" #@ LLVM Flang Fortran Compiler + "$<$:-CcdRR${RWORDSIZE}>" #@ Fujitsu HPC compiler (Trad mode) + # "$<$:>" #@ Fujitsu HPC compiler (Clang mode) + "$<$:-r${RWORDSIZE};-i4>" #@ G95 Fortran + "$<$,$>:-fdefault-real-${RWORDSIZE}>" #@ GNU Compiler Collection + # "$<$:>" #@ Green Hills Software + # "$<$:>" #@ Hewlett-Packard Compiler + # "$<$:>" #@ IAR Systems + "$<$:-real-size;${RWORDSIZE_B};-i4>" #@ Intel Classic Compiler + "$<$:-real-size;${RWORDSIZE_B};-i4>" #@ Intel LLVM-Based Compiler + # "$<$:>" #@ MCST Elbrus C/C++/Fortran Compiler + # "$<$:>" #@ Microsoft Visual Studio + "$<$:-r${RWORDSIZE};-i4>" #@ NVIDIA HPC Compiler + # "$<$:>" #@ NVIDIA CUDA Compiler + # "$<$:>" #@ Open Watcom + "$<$:-r${RWORDSIZE};-i4>" #@ The Portland Group + "$<$:-r${RWORDSIZE};-i4>" #@ PathScale + # "$<$:>" #@ Small Device C Compiler + # "$<$:>" #@ Oracle Solaris Studio + # "$<$:>" #@ Tasking Compiler Toolsets + # "$<$:>" #@ Texas Instruments + # "$<$:>" #@ Tiny C Compiler + "$<$:-qrealsize=${RWORDSIZE};-qintsize=4>" #@ IBM XL + # "$<$:>" #@ IBM Clang-based XL + # "$<$:>" #@ IBM LLVM-based Compiler + # Todo find how to handle default selection or add new compiler IDs + # unknown how to add support for sxf90 + + # line lengths + "$<$:-ffree-line-length-none>" #@ GNU Compiler Collection + ) # https://stackoverflow.com/a/53155812 @@ -593,98 +596,98 @@ add_compile_options( # Whole project flags -add_compile_options( - # $<$:-cpp> - # Use "" and ; specifically to evaluate correctly - "$<$:-diag-disable;6843>" - $<$,$>:-fallow-argument-mismatch> - $<$,$>:-fallow-invalid-boz> - $<$,$>:-ffree-line-length-none> - - # $,$:-diag-disable;6843> - ) +list( APPEND PROJECT_COMPILE_OPTIONS + # $<$:-cpp> + # Use "" and ; specifically to evaluate correctly + "$<$:-diag-disable;6843>" + $<$,$>:-fallow-argument-mismatch> + $<$,$>:-fallow-invalid-boz> + $<$,$>:-ffree-line-length-none> + + # $,$:-diag-disable;6843> + ) if ( ${PROFILE_COMPILATION} ) message( STATUS "Attemping to add compilation profiling..." ) - add_compile_options( - $<$:-ftime-report> - ) + list( APPEND PROJECT_COMPILE_OPTIONS $<$:-ftime-report> ) endif() -add_compile_definitions( - MAX_DOMAINS_F=${MAX_DOMAINS_F} - CONFIG_BUF_LEN=${CONFIG_BUF_LEN} - MAX_HISTORY=${MAX_HISTORY} - IWORDSIZE=${IWORDSIZE} - DWORDSIZE=${DWORDSIZE} - LWORDSIZE=${LWORDSIZE} - RWORDSIZE=${RWORDSIZE} +list( APPEND PROJECT_COMPILE_DEFINITIONS + MAX_DOMAINS_F=${MAX_DOMAINS_F} + CONFIG_BUF_LEN=${CONFIG_BUF_LEN} + MAX_HISTORY=${MAX_HISTORY} + IWORDSIZE=${IWORDSIZE} + DWORDSIZE=${DWORDSIZE} + LWORDSIZE=${LWORDSIZE} + RWORDSIZE=${RWORDSIZE} - NMM_MAX_DIM=2600 - NETCDF + # Alwasys set + NMM_MAX_DIM=2600 + NETCDF + + #!TODO Change this to a confcheck + NONSTANDARD_SYSTEM_SUBR + + EM_CORE=${EM_CORE} - #!TODO Change this to a confcheck - NONSTANDARD_SYSTEM_SUBR - - EM_CORE=${EM_CORE} - ) + ) # Only define if set, this is to use #ifdef/#ifndef preprocessors # in code since cmake cannot handle basically any others :( # https://gitlab.kitware.com/cmake/cmake/-/issues/17398 if ( ${ENABLE_CHEM} ) - add_compile_definitions( WRF_CHEM=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_CHEM=1 ) if ( ${ENABLE_KPP} ) - add_compile_definitions( WRF_KPP=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_KPP=1 ) endif() endif() if ( ${ENABLE_CHEM} ) - add_compile_definitions( BUILD_CHEM=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_CHEM=1 ) endif() if ( ${ENABLE_CMAQ} ) - add_compile_definitions( WRF_CMAQ=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_CMAQ=1 ) endif() if ( ${ENABLE_DFI_RADAR} ) - add_compile_definitions( WRF_DFI_RADAR=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_DFI_RADAR=1 ) endif() if ( ${ENABLE_TITAN} ) - add_compile_definitions( WRF_TITAN=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_TITAN=1 ) endif() if ( ${ENABLE_MARS} ) - add_compile_definitions( WRF_MARS=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_MARS=1 ) endif() if ( ${ENABLE_VENUS} ) - add_compile_definitions( WRF_VENUS=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_VENUS=1 ) endif() if ( ${ENABLE_HYDRO} ) - add_compile_definitions( WRF_HYDRO=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_HYDRO=1 ) endif() # Because once again we need two defines to control one thing if ( ${ENABLE_CTSM} ) - add_compile_definitions( WRF_USE_CTSM ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_USE_CTSM ) else() #!TODO there are some files that rely on this being 1, but that is never set by the legacy make system - add_compile_definitions( WRF_USE_CLM ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_USE_CLM ) endif() # If force classic or no nc-4 support enable classic if ( ${FORCE_NETCDF_CLASSIC} OR ( NOT ${netCDF_NC4} ) ) - add_compile_definitions( NETCDF_classic=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS NETCDF_classic=1 ) endif() if ( ${WRFIO_NCD_NO_LARGE_FILE_SUPPORT} OR ( NOT ${netCDF_LARGE_FILE_SUPPORT} ) ) - add_compile_definitions( WRFIO_NCD_NO_LARGE_FILE_SUPPORT=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRFIO_NCD_NO_LARGE_FILE_SUPPORT=1 ) endif() # May need a check for WRFIO_ncdpar_LARGE_FILE_SUPPORT # Now set the opposite in different defines, because why not :) if ( ( NOT ${FORCE_NETCDF_CLASSIC} ) AND ${netCDF_NC4} ) - add_compile_definitions( USE_NETCDF4_FEATURES=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS USE_NETCDF4_FEATURES=1 ) endif() if ( ( NOT ${WRFIO_NCD_NO_LARGE_FILE_SUPPORT} ) AND ${netCDF_LARGE_FILE_SUPPORT} ) - add_compile_definitions( WRFIO_NCD_LARGE_FILE_SUPPORT=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRFIO_NCD_LARGE_FILE_SUPPORT=1 ) endif() # Could simplify logic to just check if RPC is available but to be explicit @@ -692,122 +695,134 @@ endif() # not enable terran or not rpc_found do # not ( enable terrain and rpc_found ) # if ( NOT ( ${ENABLE_TERRAIN} AND ${RPC_FOUND} ) ) # this is wrong, needs fixing -# add_compile_definitions( LANDREAD_STUB ) +# list( APPEND PROJECT_COMPILE_DEFINITIONS LANDREAD_STUB ) # endif() if ( ${ENABLE_TERRAIN} AND ${MOVE_NESTS} ) - add_compile_definitions( TERRAIN_AND_LANDUSE ) + list( APPEND PROJECT_COMPILE_DEFINITIONS TERRAIN_AND_LANDUSE ) else () - add_compile_definitions( LANDREAD_STUB ) + list( APPEND PROJECT_COMPILE_DEFINITIONS LANDREAD_STUB ) endif() if ( ${USE_ALLOCATABLES} ) - add_compile_definitions( USE_ALLOCATABLES ) + list( APPEND PROJECT_COMPILE_DEFINITIONS USE_ALLOCATABLES ) endif() if ( ${wrfmodel} ) - add_compile_definitions( wrfmodel ) + list( APPEND PROJECT_COMPILE_DEFINITIONS wrfmodel ) endif() if ( ${GRIB1} ) - add_compile_definitions( GRIB1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS GRIB1 ) endif() if ( ${INTIO} ) - add_compile_definitions( INTIO ) + list( APPEND PROJECT_COMPILE_DEFINITIONS INTIO ) endif() if ( ${KEEP_INT_AROUND} ) - add_compile_definitions( KEEP_INT_AROUND ) + list( APPEND PROJECT_COMPILE_DEFINITIONS KEEP_INT_AROUND ) endif() if ( ${LIMIT_ARGS} ) - add_compile_definitions( LIMIT_ARGS ) + list( APPEND PROJECT_COMPILE_DEFINITIONS LIMIT_ARGS ) endif() if ( ${BUILD_RRTMG_FAST} ) - add_compile_definitions( BUILD_RRTMG_FAST=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_RRTMG_FAST=1 ) else() - add_compile_definitions( BUILD_RRTMG_FAST=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_RRTMG_FAST=0 ) endif() if ( ${BUILD_RRTMK} ) - add_compile_definitions( BUILD_RRTMK=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_RRTMK=1 ) else() - add_compile_definitions( BUILD_RRTMK=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_RRTMK=0 ) endif() if ( ${BUILD_SBM_FAST} ) - add_compile_definitions( BUILD_SBM_FAST=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_SBM_FAST=1 ) else() - add_compile_definitions( BUILD_SBM_FAST=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_SBM_FAST=0 ) endif() if ( ${SHOW_ALL_VARS_USED} ) - add_compile_definitions( SHOW_ALL_VARS_USED=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS SHOW_ALL_VARS_USED=1 ) else() - add_compile_definitions( SHOW_ALL_VARS_USED=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS SHOW_ALL_VARS_USED=0 ) endif() if ( ${NMM_CORE} ) - add_compile_definitions( NMM_CORE=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS NMM_CORE=1 ) else() - add_compile_definitions( NMM_CORE=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS NMM_CORE=0 ) endif() if ( "${WRF_CORE}" STREQUAL "PLUS" ) - add_compile_definitions( WRFPLUS=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRFPLUS=1 ) else() - add_compile_definitions( WRFPLUS=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS WRFPLUS=0 ) endif() if ( "${WRF_CORE}" STREQUAL "DA_CORE" OR "${WRF_CORE}" STREQUAL "DA_4D_VAR" ) - add_compile_definitions( DA_CORE=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS DA_CORE=1 ) else() - add_compile_definitions( DA_CORE=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS DA_CORE=0 ) endif() # DFI_RADAR=$ # Nesting options if ( ${MOVE_NESTS} ) - add_compile_definitions( MOVE_NESTS ) + list( APPEND PROJECT_COMPILE_DEFINITIONS MOVE_NESTS ) endif() if ( "${WRF_NESTING}" STREQUAL "VORTEX" ) - add_compile_definitions( VORTEX_CENTER ) + list( APPEND PROJECT_COMPILE_DEFINITIONS VORTEX_CENTER ) endif() # Configuration checks if ( NOT ${Fortran_2003_IEEE} ) - add_compile_definitions( NO_IEEE_MODULE ) + list( APPEND PROJECT_COMPILE_DEFINITIONS NO_IEEE_MODULE ) endif() if ( NOT ${Fortran_2003_ISO_C} ) - add_compile_definitions( NO_ISO_C_SUPPORT ) + list( APPEND PROJECT_COMPILE_DEFINITIONS NO_ISO_C_SUPPORT ) endif() # If flush fails, check if we can fall back to fflush, and if not no support if ( NOT ${Fortran_2003_FLUSH} ) if ( "${Fortran_2003_FFLUSH}" ) - add_compile_definitions( USE_FFLUSH ) + list( APPEND PROJECT_COMPILE_DEFINITIONS USE_FFLUSH ) else() - add_compile_definitions( NO_FLUSH_SUPPORT ) + list( APPEND PROJECT_COMPILE_DEFINITIONS NO_FLUSH_SUPPORT ) endif() endif() if ( NOT ${Fortran_2003_GAMMA} ) - add_compile_definitions( NO_GAMMA_SUPPORT ) + list( APPEND PROJECT_COMPILE_DEFINITIONS NO_GAMMA_SUPPORT ) endif() #!TODO Leaving as is in WRF for now but investigate why we don't do this # https://stackoverflow.com/a/1035713 # If fseeko64 succeeds, use that, else check if we can fall back to fseeko, and if not just use fseek -if ( NOT ${FSEEKO64} ) - add_compile_definitions( FSEEKO64_OK ) +if ( ${FSEEKO64} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS FSEEKO64_OK ) elseif( "${FSEEKO}" ) - add_compile_definitions( FSEEKO_OK ) + list( APPEND PROJECT_COMPILE_DEFINITIONS FSEEKO_OK ) else() - add_compile_definitions( FSEEK_OK ) + list( APPEND PROJECT_COMPILE_DEFINITIONS FSEEK_OK ) endif() # I don't believe these are used anymore... # $<$:MPI2_SUPPORT=$> # $<$:MPI2_THREAD_SUPPORT=$> - # Make core target add_library( ${PROJECT_NAME}_Core STATIC ) +target_compile_options( + ${PROJECT_NAME}_Core + PRIVATE + ${PROJECT_COMPILE_OPTIONS} + ) + + +target_compile_definitions( + ${PROJECT_NAME}_Core + PRIVATE + ${PROJECT_COMPILE_DEFINITIONS} + ) + # Supplemental to core, or rather should be, some stuff in external is legitimately part of WRF and others # are source code from truly external repositories - albeit old versions add_subdirectory( external ) diff --git a/chem/CMakeLists.txt b/chem/CMakeLists.txt index 9bfbf3d5ac..544b253256 100644 --- a/chem/CMakeLists.txt +++ b/chem/CMakeLists.txt @@ -223,4 +223,17 @@ target_link_libraries( convert_emiss PRIVATE ${PROJECT_NAME}_Core - ) \ No newline at end of file + ) + +target_compile_options( + convert_emiss + PRIVATE + ${PROJECT_COMPILE_OPTIONS} + ) + + +target_compile_definitions( + convert_emiss + PRIVATE + ${PROJECT_COMPILE_DEFINITIONS} + ) diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 7036a9debe..e52b1c3fbf 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -20,6 +20,10 @@ # Always build +# Suffice it to say everything under this is WRF-specific while also being external +add_compile_options ( "${PROJECT_COMPILE_OPTIONS}" ) +add_compile_definitions( "${PROJECT_COMPILE_DEFINITIONS}" ) + add_subdirectory( io_int ) add_subdirectory( io_grib1 ) add_subdirectory( io_grib_share ) diff --git a/external/io_adios2/CMakeLists.txt b/external/io_adios2/CMakeLists.txt index dde531a716..2d8efd61e0 100644 --- a/external/io_adios2/CMakeLists.txt +++ b/external/io_adios2/CMakeLists.txt @@ -33,14 +33,13 @@ target_include_directories( ${FOLDER_COMPILE_TARGET} # First preprocess -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) get_target_property ( FOLDER_COMPILE_TARGET_INCLUDES ${FOLDER_COMPILE_TARGET} INCLUDE_DIRECTORIES ) wrf_c_preproc_fortran( TARGET_NAME ${FOLDER_COMPILE_TARGET}_c_preproc_wrf_io OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/preproc/ EXTENSION ".f90" INCLUDES ${FOLDER_COMPILE_TARGET_INCLUDES} - DEFINITIONS ${DIR_DEFS} + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} SOURCES wrf_io.F90 ) diff --git a/external/io_netcdf/CMakeLists.txt b/external/io_netcdf/CMakeLists.txt index ac93792869..b63a1ae474 100644 --- a/external/io_netcdf/CMakeLists.txt +++ b/external/io_netcdf/CMakeLists.txt @@ -47,14 +47,13 @@ target_include_directories( # First preprocess -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) get_target_property ( FOLDER_COMPILE_TARGET_INCLUDES ${FOLDER_COMPILE_TARGET} INCLUDE_DIRECTORIES ) wrf_c_preproc_fortran( TARGET_NAME ${FOLDER_COMPILE_TARGET}_c_preproc_wrf_io OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/preproc/ EXTENSION ".f90" INCLUDES ${FOLDER_COMPILE_TARGET_INCLUDES} - DEFINITIONS ${DIR_DEFS} + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} SOURCES wrf_io.F90 ) diff --git a/external/io_netcdfpar/CMakeLists.txt b/external/io_netcdfpar/CMakeLists.txt index 8a0db9b9c9..9536023c58 100644 --- a/external/io_netcdfpar/CMakeLists.txt +++ b/external/io_netcdfpar/CMakeLists.txt @@ -47,14 +47,13 @@ target_include_directories( # First preprocess -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) get_target_property ( FOLDER_COMPILE_TARGET_INCLUDES ${FOLDER_COMPILE_TARGET} INCLUDE_DIRECTORIES ) wrf_c_preproc_fortran( TARGET_NAME ${FOLDER_COMPILE_TARGET}_c_preproc_wrf_io OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/preproc/ EXTENSION ".f90" INCLUDES ${FOLDER_COMPILE_TARGET_INCLUDES} - DEFINITIONS ${DIR_DEFS} + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} SOURCES wrf_io.F90 ) diff --git a/external/io_pnetcdf/CMakeLists.txt b/external/io_pnetcdf/CMakeLists.txt index 1717f71383..8c7c138233 100644 --- a/external/io_pnetcdf/CMakeLists.txt +++ b/external/io_pnetcdf/CMakeLists.txt @@ -35,14 +35,13 @@ target_include_directories( ${FOLDER_COMPILE_TARGET} # First preprocess -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) get_target_property ( FOLDER_COMPILE_TARGET_INCLUDES ${FOLDER_COMPILE_TARGET} INCLUDE_DIRECTORIES ) wrf_c_preproc_fortran( TARGET_NAME ${FOLDER_COMPILE_TARGET}_c_preproc_wrf_io OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/preproc/ EXTENSION ".f90" INCLUDES ${FOLDER_COMPILE_TARGET_INCLUDES} - DEFINITIONS ${DIR_DEFS} + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} SOURCES wrf_io.F90 ) diff --git a/frame/CMakeLists.txt b/frame/CMakeLists.txt index 507678afef..85264c7d5f 100644 --- a/frame/CMakeLists.txt +++ b/frame/CMakeLists.txt @@ -14,7 +14,6 @@ set( # Generate all the combinations dynamically, not a fan of this file breakdown ######################################################################################################################## set( nl_dyn_source ) -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) foreach( n RANGE 0 7 ) wrf_c_preproc_fortran( @@ -25,7 +24,7 @@ foreach( n RANGE 0 7 ) INCLUDES ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_BINARY_DIR}/inc DEPENDENCIES registry_code - DEFINITIONS ${DIR_DEFS} NNN=${n} NL_set_ROUTINES + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} NNN=${n} NL_set_ROUTINES SOURCES nl_access_routines.F ) wrf_c_preproc_fortran( @@ -36,7 +35,7 @@ foreach( n RANGE 0 7 ) INCLUDES ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_BINARY_DIR}/inc DEPENDENCIES registry_code - DEFINITIONS ${DIR_DEFS} NNN=${n} NL_get_ROUTINES + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} NNN=${n} NL_get_ROUTINES SOURCES nl_access_routines.F ) diff --git a/hydro/CMakeLists.txt b/hydro/CMakeLists.txt index ee756e71ac..63f6f59bf0 100644 --- a/hydro/CMakeLists.txt +++ b/hydro/CMakeLists.txt @@ -1,4 +1,6 @@ # additions that WRF-Hydro's top CMakeLists.txt handles +add_compile_options( ${PROJECT_COMPILE_OPTIONS} ) +add_compile_definitions( ${PROJECT_COMPILE_DEFINITIONS} ) set(CMAKE_Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/hydro/mods) add_definitions(-DMPP_LAND) if (WRF_HYDRO_NUDGING STREQUAL "1") diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 9a2f69eca6..b0ec69d7f0 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -1,6 +1,9 @@ # WRF CMake Build set( FOLDER_COMPILE_TARGETS ) +add_compile_options ( "${PROJECT_COMPILE_OPTIONS}" ) +add_compile_definitions( "${PROJECT_COMPILE_DEFINITIONS}" ) + # First make true executables if ( ${WRF_CORE} STREQUAL "PLUS" ) add_executable( diff --git a/phys/CMakeLists.txt b/phys/CMakeLists.txt index d7d85e1c12..d3df6a28e0 100644 --- a/phys/CMakeLists.txt +++ b/phys/CMakeLists.txt @@ -5,14 +5,13 @@ # Quickly preprocess some files so that cmake can understand the module dependencies # ######################################################################################################################## -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) wrf_c_preproc_fortran( TARGET_NAME module_ra_rrtmg_preproc OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/preproc/ EXTENSION ".f90" INCLUDES ${CMAKE_CURRENT_SOURCE_DIR} DEPENDENCIES registry_code - DEFINITIONS ${DIR_DEFS} + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} SOURCES module_ra_rrtmg_lwk.F module_ra_rrtmg_lwf.F module_ra_rrtmg_swk.F diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 57bff48c2c..7f07eb2539 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,4 +1,6 @@ # WRF CMake Build +add_compile_options ( "${PROJECT_COMPILE_OPTIONS}" ) +add_compile_definitions( "${PROJECT_COMPILE_DEFINITIONS}" ) #!TODO ORGANIZE THIS FOLDER set( FOLDER_COMPILE_TARGET registry ) @@ -117,10 +119,9 @@ foreach( n RANGE 0 31 ) ) endforeach() -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) wrf_expand_definitions( RESULT_VAR REGISTRY_DEFS - DEFINITIONS ${DIR_DEFS} + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} ) # How this is not a bigger thing or not resolved is beyond me From f68f0b94762d2af577a2e68c98a67f5871cda083 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:48:58 -0700 Subject: [PATCH 22/30] CMake Prepare for alternate core selections (#2086) TYPE: enhancement KEYWORDS: cmake, wrfplus, da SOURCE: internal DESCRIPTION OF CHANGES: Problem: The current CMake build only works with the em core / ARW in mind and presents a non-intuitive selection when attempting other cores. While these cores are not yet supported in the CMake build, remove these selections sets up the configuration script to be able to present a familiar set of options. Solution: Remove normally unselectable options from `configure_new` after selecting non-ARW cores. Put in place stand-in values for these unselected variables. --- CMakeLists.txt | 8 ++++++++ arch/configure_reader.py | 8 ++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0951267f9b..d3e8c4754d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,6 +127,9 @@ set( WRF_CASE_OPTIONS EM_CONVRAD EM_SQUALL2D_X EM_SQUALL2D_Y + + # No case selected, for situations where we are not compiling ARW + NONE ) set( WRF_CASE "" CACHE STRING "WRF_CASE" ) @@ -220,6 +223,11 @@ if ( NOT ${WRF_CORE} IN_LIST WRF_CORE_OPTIONS ) message( FATAL_ERROR "WRF Core option not recognized : ${WRF_CORE}" ) endif() +if ( NOT ${WRF_CORE} STREQUAL "ARW" ) + message( STATUS "All non-ARW Cores requires using pointers in grid state struct" ) + set( USE_ALLOCATABLES OFF CACHE BOOL "Required by configuration" FORCE ) +endif() + if ( NOT ${WRF_NESTING} IN_LIST WRF_NESTING_OPTIONS ) message( FATAL_ERROR "WRF Nesting option not recognized : ${WRF_NESTING}" ) endif() diff --git a/arch/configure_reader.py b/arch/configure_reader.py index fb89bfca80..2668a86f1f 100755 --- a/arch/configure_reader.py +++ b/arch/configure_reader.py @@ -575,8 +575,12 @@ def generateCMakeToolChainFile( cmakeToolChainTemplate, output, stanza, optionsD def projectSpecificOptions( options, stanzaCfg ) : coreOption = getStringOptionSelection( options.sourceCMakeFile, "WRF_CORE_OPTIONS", "WRF_CORE" ) - nestingOption = getStringOptionSelection( options.sourceCMakeFile, "WRF_NESTING_OPTIONS", "WRF_NESTING", 1 ) - caseOption = getStringOptionSelection( options.sourceCMakeFile, "WRF_CASE_OPTIONS", "WRF_CASE" ) + if coreOption == "ARW" : + nestingOption = getStringOptionSelection( options.sourceCMakeFile, "WRF_NESTING_OPTIONS", "WRF_NESTING", 1 ) + caseOption = getStringOptionSelection( options.sourceCMakeFile, "WRF_CASE_OPTIONS", "WRF_CASE" ) + else : + nestingOption = "NONE" + caseOption = "NONE" # These are yes yesValues = [ "yes", "y", "true", "1" ] From 1e96a7ec92fc20174232a55e0eb10e285e6b6016 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:51:31 -0700 Subject: [PATCH 23/30] Restructure netCDF find modules to use modern import target (#2054) TYPE: enhancement KEYWORDS: netCDF, cmake, compilation SOURCE: internal DESCRIPTION OF CHANGES: Problem: The current iteration of the cmake build uses the old-style of finding modules. This limits the usability of transitive properties as well as the native netCDF cmake build which provide imported targets. See https://cmake.org/cmake/help/latest/manual/cmake-developer.7.html#find-modules for more information on module writing. Solution: Augment the output of the provided find module to supply an imported target that fully describes the netCDF library. Additionally, in rewriting how targets in WRF reference netCDF remove unnecessary linking to the library. Ensure the provided import target and flags of the find module matches the netCDF[-Fortran].cmake.in outputs verbatim thus emulating the native netCDF build for additional robustness. LIST OF MODIFIED FILES: M CMakeLists.txt M cmake/modules/FindnetCDF-Fortran.cmake M cmake/modules/FindnetCDF.cmake M external/CMakeLists.txt M external/RSL_LITE/CMakeLists.txt M external/atm_ocn/CMakeLists.txt M external/esmf_time_f90/CMakeLists.txt M external/fftpack/fftpack5/CMakeLists.txt M external/io_adios2/CMakeLists.txt M external/io_esmf/CMakeLists.txt M external/io_grib1/CMakeLists.txt M external/io_grib1/MEL_grib1/CMakeLists.txt M external/io_grib1/WGRIB/CMakeLists.txt M external/io_grib1/grib1_util/CMakeLists.txt M external/io_grib2/bacio-1.3/CMakeLists.txt M external/io_grib2/g2lib/CMakeLists.txt M external/io_grib_share/CMakeLists.txt M external/io_netcdf/CMakeLists.txt M external/io_netcdfpar/CMakeLists.txt RELEASE NOTE: Restructure netCDF find modules to use modern import target --- CMakeLists.txt | 33 ++-- cmake/modules/FindnetCDF-Fortran.cmake | 75 +++++++-- cmake/modules/FindnetCDF.cmake | 158 +++++++++++++++--- external/CMakeLists.txt | 2 +- external/RSL_LITE/CMakeLists.txt | 6 - external/atm_ocn/CMakeLists.txt | 6 - external/esmf_time_f90/CMakeLists.txt | 6 +- external/fftpack/fftpack5/CMakeLists.txt | 12 -- external/io_adios2/CMakeLists.txt | 3 - external/io_esmf/CMakeLists.txt | 1 - external/io_grib1/CMakeLists.txt | 9 - external/io_grib1/MEL_grib1/CMakeLists.txt | 10 +- external/io_grib1/WGRIB/CMakeLists.txt | 13 -- external/io_grib1/grib1_util/CMakeLists.txt | 10 +- external/io_grib2/bacio-1.3/CMakeLists.txt | 13 -- external/io_grib2/g2lib/CMakeLists.txt | 12 -- external/io_grib_share/CMakeLists.txt | 9 - external/io_netcdf/CMakeLists.txt | 13 +- external/io_netcdfpar/CMakeLists.txt | 12 +- hydro/CMakeLists.txt | 4 +- hydro/HYDRO_drv/CMakeLists.txt | 7 +- hydro/IO/CMakeLists.txt | 7 +- hydro/Routing/CMakeLists.txt | 7 +- hydro/Routing/Reservoirs/CMakeLists.txt | 5 +- .../CMakeLists.txt | 5 +- .../Reservoirs/RFC_Forecasts/CMakeLists.txt | 5 +- hydro/nudging/CMakeLists.txt | 6 +- 27 files changed, 237 insertions(+), 212 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d3e8c4754d..126fd01661 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -433,9 +433,19 @@ endif() find_package( netCDF REQUIRED ) find_package( netCDF-Fortran REQUIRED ) -# Make use of version checking here and not in find_package for previous versions that did not use cmake -if ( ( NOT netCDF_VERSION GREATER_EQUAL "4.1.3" ) OR ( NOT netCDF-Fortran_VERSION GREATER_EQUAL "4.1.3" ) ) - message( FATAL "Please make sure NETCDF versions are 4.1.3 or later. " ) +# Make use of version checking here and not in find_package for WRF control allowing older versions if requested +if ( NOT ${FORCE_NETCDF_CLASSIC} ) + if ( NOT netCDF_VERSION GREATER_EQUAL "4.1.3" ) + set( NC4_MISSING_REASON "(version < 4.1.3)" ) + elseif( NOT ${netCDF_HAS_NC4} ) + set( NC4_MISSING_REASON "(not build with nc4)" ) + endif() + + if ( DEFINED NC4_MISSING_REASON ) + message( STATUS "netCDF found does not support NC4 features ${NC4_MISSING_REASON}, forcing use of classic netcdf" ) + message( STATUS " To disable this message, make sure netCDF used has nc4 capabilities or specify FORCE_NETCDF_CLASSIC" ) + set( FORCE_NETCDF_CLASSIC ON CACHE BOOL "Required by netCDF found" FORCE ) + endif() endif() find_package( pnetCDF QUIET ) @@ -682,7 +692,7 @@ else() endif() # If force classic or no nc-4 support enable classic -if ( ${FORCE_NETCDF_CLASSIC} OR ( NOT ${netCDF_NC4} ) ) +if ( ${FORCE_NETCDF_CLASSIC} OR ( NOT ${netCDF_HAS_NC4} ) ) list( APPEND PROJECT_COMPILE_DEFINITIONS NETCDF_classic=1 ) endif() if ( ${WRFIO_NCD_NO_LARGE_FILE_SUPPORT} OR ( NOT ${netCDF_LARGE_FILE_SUPPORT} ) ) @@ -691,7 +701,7 @@ endif() # May need a check for WRFIO_ncdpar_LARGE_FILE_SUPPORT # Now set the opposite in different defines, because why not :) -if ( ( NOT ${FORCE_NETCDF_CLASSIC} ) AND ${netCDF_NC4} ) +if ( ( NOT ${FORCE_NETCDF_CLASSIC} ) AND ${netCDF_HAS_NC4} ) list( APPEND PROJECT_COMPILE_DEFINITIONS USE_NETCDF4_FEATURES=1 ) endif() if ( ( NOT ${WRFIO_NCD_NO_LARGE_FILE_SUPPORT} ) AND ${netCDF_LARGE_FILE_SUPPORT} ) @@ -895,8 +905,6 @@ target_include_directories( ${PROJECT_SOURCE_DIR}/external/io_int # Found Packages not handled through :: imported target - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} ${pnetCDF_INCLUDE_DIRS} ) @@ -938,19 +946,10 @@ set_target_properties( EXPORT_PROPERTIES Fortran_MODULE_DIRECTORY ) -# Because of the way netCDF provides its info and the way cmake auto-gens RPATH, we need to help it along -target_link_directories( - ${PROJECT_NAME}_Core - PUBLIC - ${netCDF_LIBRARY_DIR} - ${netCDF-Fortran_LIBRARY_DIR} - ) - target_link_libraries( ${PROJECT_NAME}_Core PUBLIC - ${netCDF_LIBRARIES} - ${netCDF-Fortran_LIBRARIES} + netCDF::netcdff ${pnetCDF_LIBRARIES} $<$:$> $<$:$> diff --git a/cmake/modules/FindnetCDF-Fortran.cmake b/cmake/modules/FindnetCDF-Fortran.cmake index 0ead239a57..1b413f0721 100644 --- a/cmake/modules/FindnetCDF-Fortran.cmake +++ b/cmake/modules/FindnetCDF-Fortran.cmake @@ -43,7 +43,6 @@ else() execute_process( COMMAND ${NETCDF-FORTRAN_PROGRAM} --prefix OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF-Fortran_PREFIX ) execute_process( COMMAND ${NETCDF-FORTRAN_PROGRAM} --flibs OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF-Fortran_FLIBS ) execute_process( COMMAND ${NETCDF-FORTRAN_PROGRAM} --version OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF-Fortran_VERSION_RAW ) - execute_process( COMMAND ${NETCDF-FORTRAN_PROGRAM} --has-nc4 OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF-Fortran_NC4_YES ) # check for large file support find_file( netCDF-Fortran_INCLUDE_FILE netcdf.inc ${netCDF-Fortran_INCLUDE_DIR} ) @@ -59,31 +58,81 @@ else() string( REPLACE " " ";" netCDF-Fortran_VERSION_LIST ${netCDF-Fortran_VERSION_RAW} ) list( GET netCDF-Fortran_VERSION_LIST -1 netCDF-Fortran_VERSION ) - # Convert to YES/NO - Note cannot be generator expression if you want to use it during configuration time - string( TOUPPER ${netCDF-Fortran_NC4_YES} netCDF-Fortran_NC4 ) + # These do not pull all options available from nc-config out, but rather mirrors what is available from netCDFConfig.cmake.in + set( + netCDF-Fortran_QUERY_YES_OPTIONS + dap + nc2 + nc4 + f90 + f03 + ) + + foreach( NF_QUERY ${netCDF-Fortran_QUERY_YES_OPTIONS} ) + execute_process( COMMAND ${NETCDF-FORTRAN_PROGRAM} --has-${NF_QUERY} OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF-Fortran_${NF_QUERY}_LOWERCASE ) + if ( NOT "${netCDF-Fortran_${NF_QUERY}_LOWERCASE}" ) + # might be empty + set( netCDF-Fortran_${NF_QUERY}_LOWERCASE no ) + endif() + string( TOUPPER ${NF_QUERY} NF_QUERY_UPPERCASE ) + string( TOUPPER ${netCDF-Fortran_${NF_QUERY}_LOWERCASE} NF_ANSWER_UPPERCASE ) + # Convert to netCDF-Fortran_HAS_* = YES/NO - Note this cannot be generator expression if you want to use it during configuration time + set( netCDF-Fortran_HAS_${NF_QUERY_UPPERCASE} ${NF_ANSWER_UPPERCASE} ) + endforeach() - set( netCDF-Fortran_DEFINITIONS ) - set( netCDF-Fortran_LIBRARY_DIR ${netCDF-Fortran_PREFIX}/lib ) + # A bug in previous netcdf-fortran cmake builds, extract from flibs + string( REGEX MATCH "^-L([^ ]*)" netCDF-Fortran_LIBRARY_LINK_LOCATION ${netCDF-Fortran_FLIBS} ) + set( netCDF-Fortran_LIBRARY_DIR ${CMAKE_MATCH_1} ) + + set( netCDF-Fortran_DEFINITIONS ) set( netCDF-Fortran_LIBRARIES $<$:${netCDF-Fortran_FLIBS}> ) # Because we may need this for in-situ manual preprocessing do not use genex set( netCDF-Fortran_INCLUDE_DIRS ${netCDF-Fortran_INCLUDE_DIR} ) -endif() -find_package( PkgConfig ) + # Find the actual name of the library + find_library( + netCDF-Fortran_LIBRARY + netcdff + PATHS ${netCDF-Fortran_LIBRARY_DIR} + NO_DEFAULT_PATH + ) +endif() include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set netCDF-Fortran_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args( - netCDF-Fortran DEFAULT_MSG - netCDF-Fortran_INCLUDE_DIRS - netCDF-Fortran_FLIBS - netCDF-Fortran_VERSION - ) + netCDF-Fortran + FOUND_VAR netCDF-Fortran_FOUND + REQUIRED_VARS + netCDF-Fortran_INCLUDE_DIRS + netCDF-Fortran_LIBRARIES + netCDF-Fortran_VERSION + VERSION_VAR netCDF-Fortran_VERSION + HANDLE_VERSION_RANGE + ) + +# Note that the name of the target is the target library name as specified by the netCDF cmake build, +# NOT the netCDF repository name, I've kept this consistent to the provided netCDF builds rather +# than the convention of *_ to specify multiple components. This also helps account for the +# fact that the netCDF langauge-specific projects are separate projects +if ( netCDF-Fortran_FOUND AND NOT TARGET netCDF::netcdff ) + find_package( netCDF REQUIRED ) + + add_library( netCDF::netcdff UNKNOWN IMPORTED ) + set_target_properties( + netCDF::netcdff + PROPERTIES + IMPORTED_LOCATION "${netCDF-Fortran_LIBRARY}" + IMPORTED_LINK_INTERFACE_LANGUAGES Fortran + INTERFACE_INCLUDE_DIRECTORIES "${netCDF-Fortran_INCLUDE_DIRS}" + ) + target_link_libraries( netCDF::netcdff INTERFACE netCDF::netcdf ) +endif() -mark_as_advanced( netCDF-Fortran_FLIBS netCDF-Fortran_PREFIX netCDF-Fortran_LIBRARY_DIR ) \ No newline at end of file +mark_as_advanced( netCDF-Fortran_FLIBS netCDF-Fortran_PREFIX netCDF-Fortran_LIBRARY_DIR ) diff --git a/cmake/modules/FindnetCDF.cmake b/cmake/modules/FindnetCDF.cmake index 518ec95348..896acaba00 100644 --- a/cmake/modules/FindnetCDF.cmake +++ b/cmake/modules/FindnetCDF.cmake @@ -45,9 +45,36 @@ else() execute_process( COMMAND ${NETCDF_PROGRAM} --prefix OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_PREFIX ) execute_process( COMMAND ${NETCDF_PROGRAM} --libs OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_CLIBS ) execute_process( COMMAND ${NETCDF_PROGRAM} --version OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_VERSION_RAW ) - execute_process( COMMAND ${NETCDF_PROGRAM} --has-nc4 OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_NC4_YES ) - execute_process( COMMAND ${NETCDF_PROGRAM} --has-pnetcdf OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_PNETCDF_YES ) - execute_process( COMMAND ${NETCDF_PROGRAM} --has-parallel OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_PARALLEL_YES ) + + # These do not pull all options available from nc-config out, but rather mirrors what is available from netCDFConfig.cmake.in + set( + netCDF_QUERY_YES_OPTIONS + dap + dap2 + dap4 + nc2 + nc4 + hdf5 + hdf4 + pnetcdf + parallel + + # These are not part of the config but used in this to provide the properly linking + szlib + zstd + ) + + foreach( NC_QUERY ${netCDF_QUERY_YES_OPTIONS} ) + execute_process( COMMAND ${NETCDF_PROGRAM} --has-${NC_QUERY} OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_${NC_QUERY}_LOWERCASE ) + if ( NOT "${netCDF-Fortran_${NF_QUERY}_LOWERCASE}" ) + # might be empty + set( netCDF-Fortran_${NF_QUERY}_LOWERCASE no ) + endif() + string( TOUPPER ${NC_QUERY} NC_QUERY_UPPERCASE ) + string( TOUPPER ${netCDF_${NC_QUERY}_LOWERCASE} NC_ANSWER_UPPERCASE ) + # Convert to netCDF_HAS_* = YES/NO - Note this cannot be generator expression if you want to use it during configuration time + set( netCDF_HAS_${NC_QUERY_UPPERCASE} ${NC_ANSWER_UPPERCASE} ) + endforeach() # check for large file support find_file( netCDF_INCLUDE_FILE netcdf.h ${netCDF_INCLUDE_DIR} ) @@ -63,13 +90,7 @@ else() string( REPLACE " " ";" netCDF_VERSION_LIST ${netCDF_VERSION_RAW} ) list( GET netCDF_VERSION_LIST -1 netCDF_VERSION ) - # Convert to YES/NO - Note cannot be generator expression if you want to use it during configuration time - string( TOUPPER ${netCDF_NC4_YES} netCDF_NC4 ) - string( TOUPPER ${netCDF_PNETCDF_YES} netCDF_PNETCDF ) - string( TOUPPER ${netCDF_PARALLEL_YES} netCDF_PARALLEL ) - set( netCDF_DEFINITIONS ) - set( netCDF_LIBRARIES # All supported language variants will need this regardless - this may conflict with the RPATH in any # supplemental packages so be careful to use compatible langauge versions of netCDF @@ -77,19 +98,118 @@ else() ) # Because we may need this for in-situ manual preprocessing do not use genex set( netCDF_INCLUDE_DIRS ${netCDF_INCLUDE_DIR} ) -endif() -find_package( PkgConfig ) + # Find the actual name of the library + find_library( + netCDF_LIBRARY + netcdf + PATHS ${netCDF_LIBRARY_DIR} + NO_DEFAULT_PATH + ) + if( ${netCDF_LIBRARY} MATCHES ".a$" ) + set( netCDF_STATIC TRUE ) + set( netCDF_SHARED FALSE ) + else() + set( netCDF_STATIC FALSE ) + set( netCDF_SHARED TRUE ) + endif() +endif() include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set netCDF_FOUND to TRUE # if all listed variables are TRUE -find_package_handle_standard_args( netCDF DEFAULT_MSG - netCDF_INCLUDE_DIRS - netCDF_LIBRARY_DIR - netCDF_CLIBS - netCDF_VERSION - ) - -mark_as_advanced( netCDF_CLIBS netCDF_PREFIX netCDF_LIBRARY_DIR ) \ No newline at end of file +find_package_handle_standard_args( + netCDF + FOUND_VAR netCDF_FOUND + REQUIRED_VARS + netCDF_INCLUDE_DIRS + netCDF_LIBRARIES + netCDF_VERSION + VERSION_VAR netCDF_VERSION + HANDLE_VERSION_RANGE + ) + +# Note that the name of the target is the target library name as specified by the netCDF cmake build, +# NOT the netCDF repository name, I've kept this consistent to the provided netCDF builds rather +# than the convention of *_ to specify multiple components. This also helps account for the +# fact that the netCDF langauge-specific projects are separate projects +if ( netCDF_FOUND AND NOT TARGET netCDF::netcdf ) + add_library( netCDF::netcdf UNKNOWN IMPORTED ) + set_target_properties( + netCDF::netcdf + PROPERTIES + IMPORTED_LOCATION "${netCDF_LIBRARY}" + IMPORTED_LINK_INTERFACE_LANGUAGES C + INTERFACE_INCLUDE_DIRECTORIES "${netCDF_INCLUDE_DIRS}" + ) + + # I believe this is not required as the API will wrap usage of these + # libnetcdf should provide the rpath dependency linking if not static + # This is untested! Probably missing a lot of stuff + if ( ${netCDF_STATIC} ) + if ( ${netCDF_HAS_HDF4} OR ${netCDF_HAS_HDF5} ) + if ( ${netCDF_HAS_PARALLEL} ) + set( HDF5_PREFER_PARALLEL TRUE ) + endif() + + find_package( HDF5 COMPONENTS C HL REQUIRED ) + target_link_libraries( netCDF::netcdf INTERFACE HDF5::HDF5 ) + + find_package( ZLIB REQUIRED ) + target_link_libraries( netCDF::netcdf INTERFACE ZLIB::ZLIB ) + endif() + + # If available do find it and link in + find_package( ZLIB QUIET ) + if ( ${ZLIB_FOUND} ) + target_link_libraries( netCDF::netcdf INTERFACE ZLIB::ZLIB ) + endif() + + if ( ${netCDF_HAS_SZLIB} ) + # Currently no standard way to find szlib, and netCDF does not export its module out soooo + message( WARNING "No standard way to locate szlib, will attempt to link with -lsz" ) + target_link_libraries( netCDF::netcdf INTERFACE sz ) + endif() + + if ( ${netCDF_HAS_ZSTD} ) + find_package( PkgConfig ) + pkg_check_modules( PC_ZSTD QUIET zstd ) + target_link_libraries( netCDF::netcdf INTERFACE ${PC_ZSTD_LINK_LIBRARIES} ) + endif() + + # HAS_BZ2 is not defined right now + # if ( ${netCDF_HAS_BZ2} ) + find_package( BZip2 QUIET ) + if ( ${BZIP2_FOUND} ) + target_link_libraries( netCDF::netcdf INTERFACE BZip2::BZip2 ) + endif() + # endif() + + # HAS_LIBXML2 is not defined right now + find_package( LibXml2 QUIET ) + if( LibXml2_FOUND ) + target_link_libraries( netCDF::netcdf INTERFACE LibXml2::LibXml2 ) + endif() + + if ( ${netCDF_HAS_PARALLEL} ) + find_package( MPI COMPONENTS C REQUIRED ) + target_link_libraries( netCDF::netcdf INTERFACE MPI::MPI_C ) + endif() + + # Always linked + find_package( CURL REQUIRED ) + find_library( MATH_LIB NAMES m ) + target_link_libraries( + netCDF::netcdf + INTERFACE + ${CMAKE_DL_LIBS} + CURL::libcurl + ${MATH_LIB} + ) + endif() + +endif() + + +mark_as_advanced( netCDF_CLIBS netCDF_PREFIX netCDF_LIBRARY_DIR ) diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index e52b1c3fbf..a1fed0df77 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -54,7 +54,7 @@ add_subdirectory( esmf_time_f90 ) add_subdirectory( io_netcdf ) #!TODO We should collapse all these files into #ifdefs even if they are compiled # multiple times with different defs for the same configuration -if ( ${netCDF_PARALLEL} AND ${USE_MPI} ) +if ( ${netCDF_HAS_PARALLEL} AND ${USE_MPI} ) message( STATUS "Adding [io_netcdfpar] to configuration" ) add_subdirectory( io_netcdfpar ) endif() diff --git a/external/RSL_LITE/CMakeLists.txt b/external/RSL_LITE/CMakeLists.txt index 5f38783343..71500f040e 100644 --- a/external/RSL_LITE/CMakeLists.txt +++ b/external/RSL_LITE/CMakeLists.txt @@ -32,16 +32,10 @@ set_target_properties( target_link_libraries( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_LIBRARIES} $<$:$> $<$:$> ) -target_include_directories( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_INCLUDE_DIRS} - ) - install( TARGETS ${FOLDER_COMPILE_TARGET} EXPORT ${EXPORT_NAME}Targets diff --git a/external/atm_ocn/CMakeLists.txt b/external/atm_ocn/CMakeLists.txt index 2fe79f79d3..d3d2762ccd 100644 --- a/external/atm_ocn/CMakeLists.txt +++ b/external/atm_ocn/CMakeLists.txt @@ -28,16 +28,10 @@ set_target_properties( target_link_libraries( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_LIBRARIES} $<$:$> $<$:$> ) -target_include_directories( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_INCLUDE_DIRS} - ) - install( TARGETS ${FOLDER_COMPILE_TARGET} EXPORT ${EXPORT_NAME}Targets diff --git a/external/esmf_time_f90/CMakeLists.txt b/external/esmf_time_f90/CMakeLists.txt index 3bba5fdd69..9e3f610878 100644 --- a/external/esmf_time_f90/CMakeLists.txt +++ b/external/esmf_time_f90/CMakeLists.txt @@ -42,15 +42,13 @@ set_target_properties( target_link_libraries( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_LIBRARIES} $<$:$> $<$:$> - ) + ) target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_LIST_DIR} + ${CMAKE_CURRENT_SOURCE_DIR} ) install( diff --git a/external/fftpack/fftpack5/CMakeLists.txt b/external/fftpack/fftpack5/CMakeLists.txt index 1ae8c648de..0de0413361 100644 --- a/external/fftpack/fftpack5/CMakeLists.txt +++ b/external/fftpack/fftpack5/CMakeLists.txt @@ -32,18 +32,6 @@ set_target_properties( EXPORT_PROPERTIES Fortran_MODULE_DIRECTORY ) -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - -target_include_directories( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_INCLUDE_DIRS} - ) - install( TARGETS ${FOLDER_COMPILE_TARGET} EXPORT ${EXPORT_NAME}Targets diff --git a/external/io_adios2/CMakeLists.txt b/external/io_adios2/CMakeLists.txt index 2d8efd61e0..a63a7e7914 100644 --- a/external/io_adios2/CMakeLists.txt +++ b/external/io_adios2/CMakeLists.txt @@ -18,15 +18,12 @@ set_target_properties( target_link_libraries( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_LIBRARIES} $<$:$> $<$:$> ) target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ioapi_share ${CMAKE_INSTALL_PREFIX}/${FOLDER_COMPILE_TARGET} ) diff --git a/external/io_esmf/CMakeLists.txt b/external/io_esmf/CMakeLists.txt index 522e20bc00..d99372f3f9 100644 --- a/external/io_esmf/CMakeLists.txt +++ b/external/io_esmf/CMakeLists.txt @@ -30,7 +30,6 @@ set_target_properties( target_link_libraries( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_LIBRARIES} $<$:$> $<$:$> ) diff --git a/external/io_grib1/CMakeLists.txt b/external/io_grib1/CMakeLists.txt index c21a07be84..3c6a057d83 100644 --- a/external/io_grib1/CMakeLists.txt +++ b/external/io_grib1/CMakeLists.txt @@ -29,17 +29,8 @@ set_target_properties( ) -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ioapi_share ${CMAKE_CURRENT_SOURCE_DIR}/../io_grib_share ${CMAKE_CURRENT_SOURCE_DIR}/grib1_util diff --git a/external/io_grib1/MEL_grib1/CMakeLists.txt b/external/io_grib1/MEL_grib1/CMakeLists.txt index b275211c69..2055ace554 100644 --- a/external/io_grib1/MEL_grib1/CMakeLists.txt +++ b/external/io_grib1/MEL_grib1/CMakeLists.txt @@ -48,17 +48,9 @@ target_sources( ) -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} + # ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../../ioapi_share ) diff --git a/external/io_grib1/WGRIB/CMakeLists.txt b/external/io_grib1/WGRIB/CMakeLists.txt index 03f53648ff..4af43a5c12 100644 --- a/external/io_grib1/WGRIB/CMakeLists.txt +++ b/external/io_grib1/WGRIB/CMakeLists.txt @@ -50,19 +50,6 @@ target_sources( nceptab_131.c ) - -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - -target_include_directories( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_INCLUDE_DIRS} - ) - install( TARGETS ${FOLDER_COMPILE_TARGET} EXPORT ${EXPORT_NAME}Targets diff --git a/external/io_grib1/grib1_util/CMakeLists.txt b/external/io_grib1/grib1_util/CMakeLists.txt index c480ff8f87..e980da0152 100644 --- a/external/io_grib1/grib1_util/CMakeLists.txt +++ b/external/io_grib1/grib1_util/CMakeLists.txt @@ -16,17 +16,9 @@ target_sources( ) -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} + # ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../MEL_grib1 ) diff --git a/external/io_grib2/bacio-1.3/CMakeLists.txt b/external/io_grib2/bacio-1.3/CMakeLists.txt index 450cb510ed..28dd4afc10 100644 --- a/external/io_grib2/bacio-1.3/CMakeLists.txt +++ b/external/io_grib2/bacio-1.3/CMakeLists.txt @@ -21,19 +21,6 @@ set_target_properties( EXPORT_PROPERTIES Fortran_MODULE_DIRECTORY ) - -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - -target_include_directories( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_INCLUDE_DIRS} - ) - install( TARGETS ${FOLDER_COMPILE_TARGET} EXPORT ${EXPORT_NAME}Targets diff --git a/external/io_grib2/g2lib/CMakeLists.txt b/external/io_grib2/g2lib/CMakeLists.txt index 70246d4d16..d3599812b5 100644 --- a/external/io_grib2/g2lib/CMakeLists.txt +++ b/external/io_grib2/g2lib/CMakeLists.txt @@ -83,20 +83,8 @@ set_target_properties( ) -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - $ - ) - target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} - #!TODO Fix duplicates of wrf_[io|status]_flags.h - # ${CMAKE_CURRENT_SOURCE_DIR}/../ioapi_share ${CMAKE_CURRENT_SOURCE_DIR}/../io_grib_share ) diff --git a/external/io_grib_share/CMakeLists.txt b/external/io_grib_share/CMakeLists.txt index f62d453e8b..edc5347dd4 100644 --- a/external/io_grib_share/CMakeLists.txt +++ b/external/io_grib_share/CMakeLists.txt @@ -25,17 +25,8 @@ set_target_properties( ) -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ioapi_share ) diff --git a/external/io_netcdf/CMakeLists.txt b/external/io_netcdf/CMakeLists.txt index b63a1ae474..fee6d26252 100644 --- a/external/io_netcdf/CMakeLists.txt +++ b/external/io_netcdf/CMakeLists.txt @@ -21,26 +21,15 @@ target_link_libraries( PUBLIC $<$:$> $<$:$> - ${netCDF_LIBRARIES} - ${netCDF-Fortran_LIBRARIES} + netCDF::netcdff ) -# Because of the way netCDF provides its info and the way cmake auto-gens RPATH, we need to help it along -target_link_directories( - ${FOLDER_COMPILE_TARGET} - PUBLIC - ${netCDF_LIBRARY_DIR} - ${netCDF-Fortran_LIBRARY_DIR} - ) - target_include_directories( ${FOLDER_COMPILE_TARGET} PUBLIC $ $ $ - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/external/io_netcdfpar/CMakeLists.txt b/external/io_netcdfpar/CMakeLists.txt index 9536023c58..91a57f3665 100644 --- a/external/io_netcdfpar/CMakeLists.txt +++ b/external/io_netcdfpar/CMakeLists.txt @@ -21,17 +21,9 @@ target_link_libraries( PUBLIC $<$:$> $<$:$> - ${netCDF_LIBRARIES} - ${netCDF-Fortran_LIBRARIES} + netCDF::netcdff ) -# Because of the way netCDF provides its info and the way cmake auto-gens RPATH, we need to help it along -target_link_directories( - ${FOLDER_COMPILE_TARGET} - PUBLIC - ${netCDF_LIBRARY_DIR} - ${netCDF-Fortran_LIBRARY_DIR} - ) target_include_directories( ${FOLDER_COMPILE_TARGET} @@ -39,8 +31,6 @@ target_include_directories( $ $ $ - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/hydro/CMakeLists.txt b/hydro/CMakeLists.txt index 63f6f59bf0..5cf8615149 100644 --- a/hydro/CMakeLists.txt +++ b/hydro/CMakeLists.txt @@ -1,6 +1,6 @@ # additions that WRF-Hydro's top CMakeLists.txt handles -add_compile_options( ${PROJECT_COMPILE_OPTIONS} ) -add_compile_definitions( ${PROJECT_COMPILE_DEFINITIONS} ) +add_compile_options( "${PROJECT_COMPILE_OPTIONS}" ) +add_compile_definitions( "${PROJECT_COMPILE_DEFINITIONS}" ) set(CMAKE_Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/hydro/mods) add_definitions(-DMPP_LAND) if (WRF_HYDRO_NUDGING STREQUAL "1") diff --git a/hydro/HYDRO_drv/CMakeLists.txt b/hydro/HYDRO_drv/CMakeLists.txt index 801930bda5..e0b1a6f442 100644 --- a/hydro/HYDRO_drv/CMakeLists.txt +++ b/hydro/HYDRO_drv/CMakeLists.txt @@ -8,14 +8,11 @@ target_link_libraries(hydro_driver PUBLIC hydro_data_rec hydro_routing hydro_debug_utils + PRIVATE + netCDF::netcdff ) if(WRF_HYDRO_NUDGING STREQUAL "1") target_link_libraries(hydro_driver PUBLIC hydro_nudging) endif() -target_include_directories(hydro_driver - PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} -) diff --git a/hydro/IO/CMakeLists.txt b/hydro/IO/CMakeLists.txt index 80dc6e22a3..3566adc795 100644 --- a/hydro/IO/CMakeLists.txt +++ b/hydro/IO/CMakeLists.txt @@ -5,10 +5,5 @@ add_library(hydro_netcdf_layer STATIC target_link_libraries(hydro_netcdf_layer MPI::MPI_Fortran -) - -target_include_directories(hydro_netcdf_layer - PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} + netCDF::netcdff ) diff --git a/hydro/Routing/CMakeLists.txt b/hydro/Routing/CMakeLists.txt index 87a4b337db..307b695ec6 100644 --- a/hydro/Routing/CMakeLists.txt +++ b/hydro/Routing/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(hydro_routing STATIC target_link_libraries(hydro_routing MPI::MPI_Fortran + netCDF::netcdff hydro_mpp hydro_utils hydro_orchestrator @@ -30,9 +31,3 @@ target_link_libraries(hydro_routing hydro_data_rec hydro_routing_reservoirs_rfc ) - -target_include_directories(hydro_routing - PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} -) diff --git a/hydro/Routing/Reservoirs/CMakeLists.txt b/hydro/Routing/Reservoirs/CMakeLists.txt index 2197060090..d5d3a5fb71 100644 --- a/hydro/Routing/Reservoirs/CMakeLists.txt +++ b/hydro/Routing/Reservoirs/CMakeLists.txt @@ -5,10 +5,9 @@ add_library(hydro_routing_reservoirs STATIC module_reservoir_utilities.F90 ) -target_include_directories(hydro_routing_reservoirs +target_link_libraries(hydro_routing_reservoirs PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} + netCDF::netcdff ) add_subdirectory("Level_Pool") diff --git a/hydro/Routing/Reservoirs/Persistence_Level_Pool_Hybrid/CMakeLists.txt b/hydro/Routing/Reservoirs/Persistence_Level_Pool_Hybrid/CMakeLists.txt index a6d6b09760..5fa3405b87 100644 --- a/hydro/Routing/Reservoirs/Persistence_Level_Pool_Hybrid/CMakeLists.txt +++ b/hydro/Routing/Reservoirs/Persistence_Level_Pool_Hybrid/CMakeLists.txt @@ -6,8 +6,7 @@ add_library(hydro_routing_reservoirs_hybrid STATIC add_dependencies(hydro_routing_reservoirs_hybrid hydro_routing_reservoirs) -target_include_directories(hydro_routing_reservoirs_hybrid +target_link_libraries(hydro_routing_reservoirs_hybrid PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} + netCDF::netcdff ) diff --git a/hydro/Routing/Reservoirs/RFC_Forecasts/CMakeLists.txt b/hydro/Routing/Reservoirs/RFC_Forecasts/CMakeLists.txt index 1a1ac2bcf1..6c86815106 100644 --- a/hydro/Routing/Reservoirs/RFC_Forecasts/CMakeLists.txt +++ b/hydro/Routing/Reservoirs/RFC_Forecasts/CMakeLists.txt @@ -9,8 +9,7 @@ add_dependencies(hydro_routing_reservoirs_rfc hydro_routing_reservoirs_levelpool ) -target_include_directories(hydro_routing_reservoirs_rfc +target_link_libraries(hydro_routing_reservoirs_rfc PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} + netCDF::netcdff ) diff --git a/hydro/nudging/CMakeLists.txt b/hydro/nudging/CMakeLists.txt index c6e7b07b0f..efaa841343 100644 --- a/hydro/nudging/CMakeLists.txt +++ b/hydro/nudging/CMakeLists.txt @@ -10,10 +10,6 @@ target_link_libraries(hydro_nudging PRIVATE hydro_mpp hydro_data_rec hydro_orchestrator + netCDF::netcdff ) -target_include_directories(hydro_nudging - PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} -) From 2f844ad5bd80535443c9eed6dbdf1a7577b58c85 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Mon, 14 Oct 2024 15:52:35 -0700 Subject: [PATCH 24/30] Fix fseek test (#2055) TYPE: bug fix KEYWORDS: fseek, compilation, cmake SOURCE: internal DESCRIPTION OF CHANGES: Problem: The fseek test lacks correct syntax causing false negative reports of feature not existing when newer compiler standards disallow this. Solution: Add `int` to the main program in the fseek test. Also to add further robustness in detecting this feature, use the `-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE=1` generally *nix standard defines. LIST OF MODIFIED FILES: M CMakeLists.txt M confcheck/CMakeLists.txt M tools/fseek_test.c RELEASE NOTE: Fix fseek test --- CMakeLists.txt | 11 +++++++---- confcheck/CMakeLists.txt | 4 ++-- tools/fseek_test.c | 3 ++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 126fd01661..07a80586ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -810,10 +810,13 @@ endif() #!TODO Leaving as is in WRF for now but investigate why we don't do this # https://stackoverflow.com/a/1035713 # If fseeko64 succeeds, use that, else check if we can fall back to fseeko, and if not just use fseek -if ( ${FSEEKO64} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS FSEEKO64_OK ) -elseif( "${FSEEKO}" ) - list( APPEND PROJECT_COMPILE_DEFINITIONS FSEEKO_OK ) +if ( "${FSEEKO64}" OR "${FSEEKO}" ) + list( APPEND PROJECT_COMPILE_DEFINITIONS _FILE_OFFSET_BITS=64 _LARGEFILE_SOURCE=1 ) + if ( "${FSEEKO64}" ) + list( APPEND PROJECT_COMPILE_DEFINITIONS FSEEKO64_OK ) + elseif( "${FSEEKO}" ) + list( APPEND PROJECT_COMPILE_DEFINITIONS FSEEKO_OK ) + endif() else() list( APPEND PROJECT_COMPILE_DEFINITIONS FSEEK_OK ) endif() diff --git a/confcheck/CMakeLists.txt b/confcheck/CMakeLists.txt index 152aeeaa3a..aab2e3bc69 100644 --- a/confcheck/CMakeLists.txt +++ b/confcheck/CMakeLists.txt @@ -49,7 +49,7 @@ wrf_conf_check( RESULT_VAR FSEEKO64 SOURCE ${PROJECT_SOURCE_DIR}/tools/fseek_test.c EXTENSION .c - ADDITIONAL_DEFINITIONS -DTEST_FSEEKO64 -DFILE_TO_TEST="${PROJECT_SOURCE_DIR}/CMakeLists.txt" + ADDITIONAL_DEFINITIONS -DTEST_FSEEKO64 -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE=1 -DFILE_TO_TEST="${PROJECT_SOURCE_DIR}/CMakeLists.txt" MESSAGE "fseeko64 not supported, checking alternate fseeko" ) @@ -60,7 +60,7 @@ if ( NOT "${FSEEKO64}" ) RESULT_VAR FSEEKO SOURCE ${PROJECT_SOURCE_DIR}/tools/fseek_test.c EXTENSION .c - ADDITIONAL_DEFINITIONS -DTEST_FSEEKO -DFILE_TO_TEST="${PROJECT_SOURCE_DIR}/CMakeLists.txt" + ADDITIONAL_DEFINITIONS -DTEST_FSEEKO -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE=1 -DFILE_TO_TEST="${PROJECT_SOURCE_DIR}/CMakeLists.txt" MESSAGE "fseeko not supported, compiling with fseek (caution with large files)" ) endif() diff --git a/tools/fseek_test.c b/tools/fseek_test.c index c1bee099b5..45cccef22f 100644 --- a/tools/fseek_test.c +++ b/tools/fseek_test.c @@ -6,6 +6,7 @@ #include #include #include +int main() { FILE *fp ; @@ -43,6 +44,6 @@ main() retval = 1 ; } fclose(fp) ; - exit(retval) ; + return retval ; } From 5d9beb1e62cce855bd7afa45878b64df31009ed7 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Mon, 14 Oct 2024 16:59:20 -0700 Subject: [PATCH 25/30] CMake improve configure_reader.py script (#2087) TYPE: enhancement KEYWORDS: cmake, configuration SOURCE: internal DESCRIPTION OF CHANGES: Problem: In preparation for alternate core compilation, there is need for additional information from the stanzas. Some of these fields include extraneous information to cmake that must be filtered out. The previous version manually listed fields and values to sanitize, but this is brittle when accounting for all stanza types and configurations. Solution: Rather than hard-coding the sanitization of stanza fields to be consumed by cmake, a generalized approach is used to allow full use of stanza fields down the line. --- arch/configure_reader.py | 52 ++++------------------------------------ 1 file changed, 5 insertions(+), 47 deletions(-) diff --git a/arch/configure_reader.py b/arch/configure_reader.py index 2668a86f1f..98f758e0b8 100755 --- a/arch/configure_reader.py +++ b/arch/configure_reader.py @@ -17,7 +17,8 @@ osAndArchAlt = re.compile( r"^ARCH[ ]+(\w+)[ ]+(\w+)", re.I ) referenceVar = re.compile( r"[$]([(])?(\w+)(?(1)[)])", re.I ) -compileObject = re.compile( r"(\W)-c(\W)" ) +compileObject = re.compile( r"(\W|^)-c(\W|$)" ) +configureRepl = re.compile( r"(\W|^)CONFIGURE_\w+(\W|$)" ) class Stanza(): @@ -160,52 +161,9 @@ def sanitize( self ) : self.dereference( "FCBASEOPTS" ) # Remove rogue compile commands that should *NOT* even be here - keysToSanitize = [ - "ARFLAGS","ARFLAGS", - "CC", - "CFLAGS_LOCAL", - "CFLAGS", - "COMPRESSION_INC", - "COMPRESSION_LIBS", - "CPP", - "CPPFLAGS", - "DM_CC", - "DM_FC", - "ESMF_LDFLAG", - "F77FLAGS", - "FC", - "FCBASEOPTS_NO_G", - "FCBASEOPTS", - "FCOPTIM", - "FCSUFFIX", - "FDEFS", - "FFLAGS", - "FNGFLAGS", - "FORMAT_FIXED", - "FORMAT_FREE", - "LD", - "LDFLAGS_LOCAL", - "LDFLAGS", - "MODULE_SRCH_FLAG", - "RLFLAGS", - "SCC", - "SFC", - "TRADFLAG", - ] - - for keyToSan in keysToSanitize : - if keyToSan in self.kvPairs_ : - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_COMP_L", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_COMP_I", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_FC", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_CC", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_FDEFS", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_MPI", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_COMPAT_FLAGS", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_CPPFLAGS", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_TRADFLAG", "" ) - - self.kvPairs_[ keyToSan ] = compileObject.sub( r"\1\2", self.kvPairs_[ keyToSan ] ).strip() + for keyToSan in self.kvPairs_.keys() : + self.kvPairs_[ keyToSan ] = configureRepl.sub( r"\1\2", self.kvPairs_[ keyToSan ] ).strip() + self.kvPairs_[ keyToSan ] = compileObject.sub( r"\1\2", self.kvPairs_[ keyToSan ] ).strip() # Now fix certain ones that are mixing programs with flags all mashed into one option From 915f93d2381578ab43decea04d05bf23cc956603 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Tue, 15 Oct 2024 11:57:27 -0700 Subject: [PATCH 26/30] CMake compilation flags as categorical properties (#2088) TYPE: enhancement KEYWORDS: cmake, compilation, flags SOURCE: internal DESCRIPTION OF CHANGES: Problem: In preparation for alternative core selection, there is a broader range of flags that must be set and controlled throughout the project. This includes being able to set flags on a per-source basis to override certain flags that are normally applied wholly to all source within a CMake target. An example of this is disabling double precision on an individual file when that option is being used. Solution: Categorizing the flags first allows for easy application and selection of particular sets of flags for the multiple libraries and binaries created. Second, for the core library, using pseudo-inherited CMake properties between targets and source files allows for granular control of these flag categories. --- CMakeLists.txt | 348 +++++++++++++++++---------- arch/configure_reader.py | 3 +- cmake/c_preproc.cmake | 5 +- cmake/target_source_properties.cmake | 142 +++++++++++ cmake/template/arch_config.cmake | 21 +- frame/CMakeLists.txt | 2 +- 6 files changed, 378 insertions(+), 143 deletions(-) create mode 100644 cmake/target_source_properties.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 07a80586ba..3cd0264b72 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,12 +26,15 @@ list( APPEND CMAKE_C_PREPROCESSOR_FLAGS -P -nostdinc -traditional ) include( CMakePackageConfigHelpers ) include( CheckIPOSupported ) + +# WRF helper scripts include( c_preproc ) include( m4_preproc ) include( target_copy ) include( confcheck ) include( gitinfo ) include( printOption ) +include( target_source_properties ) include( wrf_case_setup ) include( wrf_get_version ) @@ -311,6 +314,51 @@ if ( ${USE_IPO} ) endif() endif() + +################################################################################ +## +## Create our flags / defines properties and variables to carry our information +## +################################################################################ +define_target_source_properties( + PROPERTIES + COMPILE_OPTIONS_OPTIMIZATION # Control of optimization flags + COMPILE_OPTIONS_CONFCHECK # Originate from system configuration checks + COMPILE_OPTIONS_OPTIONS # Derived from options + COMPILE_OPTIONS_Fortran_PRECISION # Fortran floating point precision control + COMPILE_OPTIONS_Fortran_LINE_LENGTH # Fortran line length for compilers that need it + COMPILE_OPTIONS_DIAGNOSTICS # Extra flags for diagnostics + COMPILE_OPTIONS_SUPPRESS_ERRORS # EXPLICITLY suppress errors, USE SPARINGLY!!! + COMPILE_OPTIONS_SUPPRESS_WARNINGS # Suppress warnings, try to fix before they become errs + COMPILE_OPTIONS_STANZA # Any remaining flags imported via stanza + + COMPILE_DEFINITIONS_CONFCHECK # Originate from system configuration checks + COMPILE_DEFINITIONS_OPTIONS # Anything that is derived from options + COMPILE_DEFINITIONS_STANZA # Any remaining defines imported via stanza + ) + +# These will be the cumulative set for quicker reference +set( PROJECT_COMPILE_OPTIONS "" ) +set( PROJECT_COMPILE_DEFINITIONS "" ) + +# Recreate these as project variables so that they may be re-used generically +set( PROJECT_COMPILE_OPTIONS_OPTIMIZATION "" ) +set( PROJECT_COMPILE_OPTIONS_CONFCHECK "" ) +set( PROJECT_COMPILE_OPTIONS_OPTIONS "" ) +set( PROJECT_COMPILE_OPTIONS_Fortran_PRECISION "" ) +set( PROJECT_COMPILE_OPTIONS_Fortran_LINE_LENGTH "" ) +set( PROJECT_COMPILE_OPTIONS_DIAGNOSTICS "" ) +set( PROJECT_COMPILE_OPTIONS_SUPPRESS_ERRORS "" ) +set( PROJECT_COMPILE_OPTIONS_SUPPRESS_WARNINGS "" ) +set( PROJECT_COMPILE_OPTIONS_STANZA "" ) +set( PROJECT_COMPILE_DEFINITIONS_CONFCHECK "" ) +set( PROJECT_COMPILE_DEFINITIONS_OPTIONS "" ) +set( PROJECT_COMPILE_DEFINITIONS_STANZA ${WRF_ARCH_LOCAL} ) + +# Make these imported variables easier to use in genexp +separate_arguments( WRF_FCOPTIM ) +separate_arguments( WRF_FCNOOPT ) + ################################################################################ ## ## Now find packages that cross-compilation is potentially handled @@ -330,19 +378,19 @@ if ( ${USE_MPI} ) # It still technically finds MPI but the output is nonintuitive # saying things like hdf5 or pthread find_package( MPI REQUIRED COMPONENTS Fortran C ) - list( APPEND PROJECT_COMPILE_DEFINITIONS + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS USE_MPI=1 DM_PARALLEL ) if ( DEFINED WRF_MPI_Fortran_FLAGS AND NOT "${WRF_MPI_Fortran_FLAGS}" STREQUAL "" ) - list( APPEND PROJECT_COMPILE_OPTIONS + list( APPEND PROJECT_COMPILE_OPTIONS_OPTIONS $<$:${WRF_MPI_Fortran_FLAGS}> ) endif() if ( DEFINED WRF_MPI_C_FLAGS AND NOT "${WRF_MPI_C_FLAGS}" STREQUAL "" ) - list( APPEND PROJECT_COMPILE_OPTIONS + list( APPEND PROJECT_COMPILE_OPTIONS_OPTIONS $<$:${WRF_MPI_C_FLAGS}> ) endif() @@ -373,7 +421,7 @@ if ( ${USE_MPI} ) set( USE_RSL_LITE ON ) # We know NONE is the zero index so compare against that elseif( ${CURRENT_NESTING_IDX} GREATER 0 ) - list( APPEND PROJECT_COMPILE_DEFINITIONS + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS DM_PARALLEL STUBMPI ) @@ -382,7 +430,7 @@ endif() if ( ${USE_OPENMP} ) find_package( OpenMP REQUIRED COMPONENTS Fortran C ) - list( APPEND PROJECT_COMPILE_DEFINITIONS + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS USE_OPENMP=1 SM_PARALLEL ) @@ -525,44 +573,18 @@ add_subdirectory( confcheck ) ## ################################################################################ -# https://stackoverflow.com/a/47927921 -# Define compile options to be inherited for directories -define_property( - SOURCE - PROPERTY COMPILE_FLAGS - INHERITED - BRIEF_DOCS "brief-doc" - FULL_DOCS "full-doc" - ) - -define_property( - DIRECTORY - PROPERTY COMPILE_FLAGS - INHERITED - BRIEF_DOCS "brief-doc" - FULL_DOCS "full-doc" - ) - # Get current build type flags and put them in there if ( "${CMAKE_BUILD_TYPE}" STREQUAL "Release" ) - set_directory_properties( - PROPERTIES - COMPILE_FLAGS - $<$:${WRF_FCOPTIM}> - ) + set( PROJECT_COMPILE_OPTIONS_OPTIMIZATION $<$:${WRF_FCOPTIM}> ) # else() # # Assume no optimization -# set_directory_properties( -# PROPERTIES -# COMPILE_FLAGS -# $<$:${WRF_FCNOOPT}> -# ) +# set( PROJECT_COMPILE_OPTIONS_OPTIMIZATION $<$:${WRF_FCNOOPT}> ) endif() # This is really ugly but such is the cost of supporting many ways to say the same thing # https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_COMPILER_ID.html -list( APPEND PROJECT_COMPILE_OPTIONS +list( APPEND PROJECT_COMPILE_OPTIONS_Fortran_PRECISION # Use "" and ; specifically to evaluate correctly # "$<$:>" #@ Absoft Fortran # "$<$:>" #@ Analog VisualDSP++ @@ -602,7 +624,9 @@ list( APPEND PROJECT_COMPILE_OPTIONS # "$<$:>" #@ IBM LLVM-based Compiler # Todo find how to handle default selection or add new compiler IDs # unknown how to add support for sxf90 + ) +list( APPEND PROJECT_COMPILE_OPTIONS_Fortran_LINE_LENGTH # line lengths "$<$:-ffree-line-length-none>" #@ GNU Compiler Collection ) @@ -613,24 +637,23 @@ list( APPEND PROJECT_COMPILE_OPTIONS # message( STATUS "Set Fortran_COMPILER_ID to : ${Fortran_COMPILER_ID}" ) -# Whole project flags -list( APPEND PROJECT_COMPILE_OPTIONS - # $<$:-cpp> +list( APPEND PROJECT_COMPILE_OPTIONS_SUPPRESS_WARNINGS # Use "" and ; specifically to evaluate correctly "$<$:-diag-disable;6843>" + ) + +list( APPEND PROJECT_COMPILE_OPTIONS_SUPPRESS_ERRORS $<$,$>:-fallow-argument-mismatch> $<$,$>:-fallow-invalid-boz> $<$,$>:-ffree-line-length-none> - - # $,$:-diag-disable;6843> ) if ( ${PROFILE_COMPILATION} ) message( STATUS "Attemping to add compilation profiling..." ) - list( APPEND PROJECT_COMPILE_OPTIONS $<$:-ftime-report> ) + list( APPEND PROJECT_COMPILE_OPTIONS_DIAGNOSTICS $<$:-ftime-report> ) endif() -list( APPEND PROJECT_COMPILE_DEFINITIONS +list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS MAX_DOMAINS_F=${MAX_DOMAINS_F} CONFIG_BUF_LEN=${CONFIG_BUF_LEN} MAX_HISTORY=${MAX_HISTORY} @@ -645,7 +668,7 @@ list( APPEND PROJECT_COMPILE_DEFINITIONS NETCDF #!TODO Change this to a confcheck - NONSTANDARD_SYSTEM_SUBR + # NONSTANDARD_SYSTEM_SUBR # For now let this come from stanza EM_CORE=${EM_CORE} @@ -656,56 +679,56 @@ list( APPEND PROJECT_COMPILE_DEFINITIONS # in code since cmake cannot handle basically any others :( # https://gitlab.kitware.com/cmake/cmake/-/issues/17398 if ( ${ENABLE_CHEM} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_CHEM=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_CHEM=1 ) if ( ${ENABLE_KPP} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_KPP=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_KPP=1 ) endif() endif() if ( ${ENABLE_CHEM} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_CHEM=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_CHEM=1 ) endif() if ( ${ENABLE_CMAQ} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_CMAQ=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_CMAQ=1 ) endif() if ( ${ENABLE_DFI_RADAR} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_DFI_RADAR=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_DFI_RADAR=1 ) endif() if ( ${ENABLE_TITAN} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_TITAN=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_TITAN=1 ) endif() if ( ${ENABLE_MARS} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_MARS=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_MARS=1 ) endif() if ( ${ENABLE_VENUS} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_VENUS=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_VENUS=1 ) endif() if ( ${ENABLE_HYDRO} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_HYDRO=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_HYDRO=1 ) endif() # Because once again we need two defines to control one thing if ( ${ENABLE_CTSM} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_USE_CTSM ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_USE_CTSM ) else() #!TODO there are some files that rely on this being 1, but that is never set by the legacy make system - list( APPEND PROJECT_COMPILE_DEFINITIONS WRF_USE_CLM ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_USE_CLM ) endif() # If force classic or no nc-4 support enable classic if ( ${FORCE_NETCDF_CLASSIC} OR ( NOT ${netCDF_HAS_NC4} ) ) - list( APPEND PROJECT_COMPILE_DEFINITIONS NETCDF_classic=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS NETCDF_classic=1 ) endif() if ( ${WRFIO_NCD_NO_LARGE_FILE_SUPPORT} OR ( NOT ${netCDF_LARGE_FILE_SUPPORT} ) ) - list( APPEND PROJECT_COMPILE_DEFINITIONS WRFIO_NCD_NO_LARGE_FILE_SUPPORT=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRFIO_NCD_NO_LARGE_FILE_SUPPORT=1 ) endif() # May need a check for WRFIO_ncdpar_LARGE_FILE_SUPPORT # Now set the opposite in different defines, because why not :) if ( ( NOT ${FORCE_NETCDF_CLASSIC} ) AND ${netCDF_HAS_NC4} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS USE_NETCDF4_FEATURES=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS USE_NETCDF4_FEATURES=1 ) endif() if ( ( NOT ${WRFIO_NCD_NO_LARGE_FILE_SUPPORT} ) AND ${netCDF_LARGE_FILE_SUPPORT} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS WRFIO_NCD_LARGE_FILE_SUPPORT=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRFIO_NCD_LARGE_FILE_SUPPORT=1 ) endif() # Could simplify logic to just check if RPC is available but to be explicit @@ -713,136 +736,165 @@ endif() # not enable terran or not rpc_found do # not ( enable terrain and rpc_found ) # if ( NOT ( ${ENABLE_TERRAIN} AND ${RPC_FOUND} ) ) # this is wrong, needs fixing -# list( APPEND PROJECT_COMPILE_DEFINITIONS LANDREAD_STUB ) +# list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS LANDREAD_STUB ) # endif() if ( ${ENABLE_TERRAIN} AND ${MOVE_NESTS} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS TERRAIN_AND_LANDUSE ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS TERRAIN_AND_LANDUSE ) else () - list( APPEND PROJECT_COMPILE_DEFINITIONS LANDREAD_STUB ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS LANDREAD_STUB ) endif() if ( ${USE_ALLOCATABLES} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS USE_ALLOCATABLES ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS USE_ALLOCATABLES ) endif() if ( ${wrfmodel} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS wrfmodel ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS wrfmodel ) endif() if ( ${GRIB1} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS GRIB1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS GRIB1 ) endif() if ( ${INTIO} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS INTIO ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS INTIO ) endif() if ( ${KEEP_INT_AROUND} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS KEEP_INT_AROUND ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS KEEP_INT_AROUND ) endif() if ( ${LIMIT_ARGS} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS LIMIT_ARGS ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS LIMIT_ARGS ) endif() if ( ${BUILD_RRTMG_FAST} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_RRTMG_FAST=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_RRTMG_FAST=1 ) else() - list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_RRTMG_FAST=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_RRTMG_FAST=0 ) endif() if ( ${BUILD_RRTMK} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_RRTMK=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_RRTMK=1 ) else() - list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_RRTMK=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_RRTMK=0 ) endif() if ( ${BUILD_SBM_FAST} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_SBM_FAST=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_SBM_FAST=1 ) else() - list( APPEND PROJECT_COMPILE_DEFINITIONS BUILD_SBM_FAST=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_SBM_FAST=0 ) endif() if ( ${SHOW_ALL_VARS_USED} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS SHOW_ALL_VARS_USED=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS SHOW_ALL_VARS_USED=1 ) else() - list( APPEND PROJECT_COMPILE_DEFINITIONS SHOW_ALL_VARS_USED=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS SHOW_ALL_VARS_USED=0 ) endif() if ( ${NMM_CORE} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS NMM_CORE=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS NMM_CORE=1 ) else() - list( APPEND PROJECT_COMPILE_DEFINITIONS NMM_CORE=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS NMM_CORE=0 ) endif() if ( "${WRF_CORE}" STREQUAL "PLUS" ) - list( APPEND PROJECT_COMPILE_DEFINITIONS WRFPLUS=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRFPLUS=1 ) else() - list( APPEND PROJECT_COMPILE_DEFINITIONS WRFPLUS=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRFPLUS=0 ) endif() if ( "${WRF_CORE}" STREQUAL "DA_CORE" OR "${WRF_CORE}" STREQUAL "DA_4D_VAR" ) - list( APPEND PROJECT_COMPILE_DEFINITIONS DA_CORE=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS DA_CORE=1 ) else() - list( APPEND PROJECT_COMPILE_DEFINITIONS DA_CORE=0 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS DA_CORE=0 ) endif() # DFI_RADAR=$ # Nesting options if ( ${MOVE_NESTS} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS MOVE_NESTS ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS MOVE_NESTS ) endif() if ( "${WRF_NESTING}" STREQUAL "VORTEX" ) - list( APPEND PROJECT_COMPILE_DEFINITIONS VORTEX_CENTER ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS VORTEX_CENTER ) endif() # Configuration checks if ( NOT ${Fortran_2003_IEEE} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS NO_IEEE_MODULE ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK NO_IEEE_MODULE ) endif() if ( NOT ${Fortran_2003_ISO_C} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS NO_ISO_C_SUPPORT ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK NO_ISO_C_SUPPORT ) endif() # If flush fails, check if we can fall back to fflush, and if not no support if ( NOT ${Fortran_2003_FLUSH} ) if ( "${Fortran_2003_FFLUSH}" ) - list( APPEND PROJECT_COMPILE_DEFINITIONS USE_FFLUSH ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK USE_FFLUSH ) else() - list( APPEND PROJECT_COMPILE_DEFINITIONS NO_FLUSH_SUPPORT ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK NO_FLUSH_SUPPORT ) endif() endif() if ( NOT ${Fortran_2003_GAMMA} ) - list( APPEND PROJECT_COMPILE_DEFINITIONS NO_GAMMA_SUPPORT ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK NO_GAMMA_SUPPORT ) endif() #!TODO Leaving as is in WRF for now but investigate why we don't do this # https://stackoverflow.com/a/1035713 # If fseeko64 succeeds, use that, else check if we can fall back to fseeko, and if not just use fseek if ( "${FSEEKO64}" OR "${FSEEKO}" ) - list( APPEND PROJECT_COMPILE_DEFINITIONS _FILE_OFFSET_BITS=64 _LARGEFILE_SOURCE=1 ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK _FILE_OFFSET_BITS=64 _LARGEFILE_SOURCE=1 ) if ( "${FSEEKO64}" ) - list( APPEND PROJECT_COMPILE_DEFINITIONS FSEEKO64_OK ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK FSEEKO64_OK ) elseif( "${FSEEKO}" ) - list( APPEND PROJECT_COMPILE_DEFINITIONS FSEEKO_OK ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK FSEEKO_OK ) endif() else() - list( APPEND PROJECT_COMPILE_DEFINITIONS FSEEK_OK ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK FSEEK_OK ) endif() # I don't believe these are used anymore... # $<$:MPI2_SUPPORT=$> # $<$:MPI2_THREAD_SUPPORT=$> +# Accumulate option and definitions +list( APPEND PROJECT_COMPILE_OPTIONS + ${PROJECT_COMPILE_OPTIONS_OPTIMIZATION} + ${PROJECT_COMPILE_OPTIONS_CONFCHECK} + ${PROJECT_COMPILE_OPTIONS_OPTIONS} + ${PROJECT_COMPILE_OPTIONS_Fortran_PRECISION} + ${PROJECT_COMPILE_OPTIONS_Fortran_LINE_LENGTH} + ${PROJECT_COMPILE_OPTIONS_DIAGNOSTICS} + ${PROJECT_COMPILE_OPTIONS_SUPPRESS_ERRORS} + ${PROJECT_COMPILE_OPTIONS_SUPPRESS_WARNINGS} + ${PROJECT_COMPILE_OPTIONS_STANZA} + ) +list( APPEND PROJECT_COMPILE_DEFINITIONS + ${PROJECT_COMPILE_DEFINITIONS_CONFCHECK} + ${PROJECT_COMPILE_DEFINITIONS_OPTIONS} + ${PROJECT_COMPILE_DEFINITIONS_STANZA} + ) + +################################################################################ +## +## DO NOT modify PROJECT_COMPILE_* beyond this point +## +################################################################################ + + # Make core target add_library( ${PROJECT_NAME}_Core STATIC ) -target_compile_options( - ${PROJECT_NAME}_Core - PRIVATE - ${PROJECT_COMPILE_OPTIONS} - ) - +set_target_properties( + ${PROJECT_NAME}_Core + PROPERTIES + COMPILE_OPTIONS_OPTIMIZATION "${PROJECT_COMPILE_OPTIONS_OPTIMIZATION}" + COMPILE_OPTIONS_CONFCHECK "${PROJECT_COMPILE_OPTIONS_CONFCHECK}" + COMPILE_OPTIONS_OPTIONS "${PROJECT_COMPILE_OPTIONS_OPTIONS}" + COMPILE_OPTIONS_Fortran_PRECISION "${PROJECT_COMPILE_OPTIONS_Fortran_PRECISION}" + COMPILE_OPTIONS_Fortran_LINE_LENGTH "${PROJECT_COMPILE_OPTIONS_Fortran_LINE_LENGTH}" + COMPILE_OPTIONS_DIAGNOSTICS "${PROJECT_COMPILE_OPTIONS_DIAGNOSTICS}" + COMPILE_OPTIONS_SUPPRESS_ERRORS "${PROJECT_COMPILE_OPTIONS_SUPPRESS_ERRORS}" + COMPILE_OPTIONS_SUPPRESS_WARNINGS "${PROJECT_COMPILE_OPTIONS_SUPPRESS_WARNINGS}" + COMPILE_OPTIONS_STANZA "${PROJECT_COMPILE_OPTIONS_STANZA}" + # We do not use compile definitions for CMake's lack of source file accounting + ) -target_compile_definitions( - ${PROJECT_NAME}_Core - PRIVATE - ${PROJECT_COMPILE_DEFINITIONS} - ) +# Do NOT apply these options or defines just yet, allow files to be added and set their own properties # Supplemental to core, or rather should be, some stuff in external is legitimately part of WRF and others # are source code from truly external repositories - albeit old versions @@ -855,6 +907,68 @@ add_dependencies( # So many things depend on this that I'm adding a dep here registry_code ) + +# Add directly to core +add_subdirectory( phys ) +add_subdirectory( share ) +add_subdirectory( frame ) +add_subdirectory( inc ) + +if ( ${WRF_CHEM} ) + add_subdirectory( chem ) +endif() + +if ( ${ENABLE_HYDRO} ) + add_subdirectory( hydro ) +endif() + +add_subdirectory( dyn_em ) + + +add_subdirectory( main ) + +################################################################################ +# Add subdirectory with case info +################################################################################ +if ( ${CURRENT_WRF_CORE_IDX} GREATER_EQUAL ${START_DA_IDX} ) + message( STATUS "DA or PLUS build, WRF_CASE selection ignored" ) +else() + add_subdirectory( test/${WRF_CASE_FOLDER} ) +endif() + + + +################################################################################ +## +## DO NOT add sources beyond this point +## +################################################################################ + +# gather compile info and apply +apply_target_source_properties( + TARGETS ${PROJECT_NAME}_Core + PROPERTIES + COMPILE_OPTIONS_OPTIMIZATION + COMPILE_OPTIONS_CONFCHECK + COMPILE_OPTIONS_OPTIONS + COMPILE_OPTIONS_Fortran_PRECISION + COMPILE_OPTIONS_Fortran_LINE_LENGTH + COMPILE_OPTIONS_DIAGNOSTICS + COMPILE_OPTIONS_SUPPRESS_ERRORS + COMPILE_OPTIONS_SUPPRESS_WARNINGS + COMPILE_OPTIONS_STANZA + AS_PROPERTY COMPILE_OPTIONS + ) + +target_compile_definitions( + ${PROJECT_NAME}_Core + PRIVATE + ${PROJECT_COMPILE_DEFINITIONS_CONFCHECK} + ${PROJECT_COMPILE_DEFINITIONS_OPTIONS} + ${PROJECT_COMPILE_DEFINITIONS_STANZA} + ) + + target_include_directories( ${PROJECT_NAME}_Core PUBLIC @@ -911,34 +1025,6 @@ target_include_directories( ${pnetCDF_INCLUDE_DIRS} ) -# Add directly to core -add_subdirectory( phys ) -add_subdirectory( share ) -add_subdirectory( frame ) -add_subdirectory( inc ) - -if ( ${WRF_CHEM} ) - add_subdirectory( chem ) -endif() - -if ( ${ENABLE_HYDRO} ) - add_subdirectory( hydro ) -endif() - -add_subdirectory( dyn_em ) - - -add_subdirectory( main ) - -################################################################################ -# Add subdirectory with case info -################################################################################ -if ( ${CURRENT_WRF_CORE_IDX} GREATER_EQUAL ${START_DA_IDX} ) - message( STATUS "DA or PLUS build, WRF_CASE selection ignored" ) -else() - add_subdirectory( test/${WRF_CASE_FOLDER} ) -endif() - # Configure core set_target_properties( ${PROJECT_NAME}_Core @@ -949,6 +1035,8 @@ set_target_properties( EXPORT_PROPERTIES Fortran_MODULE_DIRECTORY ) +target_link_options( ${PROJECT_NAME}_Core PRIVATE ${WRF_LINK_FLAGS} ) + target_link_libraries( ${PROJECT_NAME}_Core PUBLIC diff --git a/arch/configure_reader.py b/arch/configure_reader.py index 98f758e0b8..c52d776af7 100755 --- a/arch/configure_reader.py +++ b/arch/configure_reader.py @@ -504,6 +504,7 @@ def generateCMakeToolChainFile( cmakeToolChainTemplate, output, stanza, optionsD configStanza = cmakeToolChainTemplateLines.format( ARCH_LOCAL=stanza.kvPairs_["ARCH_LOCAL"], + LDFLAGS_LOCAL=stanza.kvPairs_["LDFLAGS_LOCAL"], BYTESWAPIO=stanza.kvPairs_["BYTESWAPIO"], CFLAGS_LOCAL=stanza.kvPairs_["CFLAGS_LOCAL"], DM_CC=stanza.kvPairs_["DM_CC"], @@ -587,4 +588,4 @@ def projectSpecificOptions( options, stanzaCfg ) : return additionalOptions if __name__ == '__main__' : - main() \ No newline at end of file + main() diff --git a/cmake/c_preproc.cmake b/cmake/c_preproc.cmake index 4de0a1e7a1..0d0dd751ef 100644 --- a/cmake/c_preproc.cmake +++ b/cmake/c_preproc.cmake @@ -117,9 +117,12 @@ macro( wrf_expand_definitions ) else() list( APPEND WRF_EXP_DEFS -D${WRF_EXP_DEF} ) endif() + else() + # Just add it normally + list( APPEND WRF_EXP_DEFS ${WRF_EXP_DEF} ) endif() endforeach() set( ${WRF_EXP_RESULT_VAR} ${WRF_EXP_DEFS} ) -endmacro() \ No newline at end of file +endmacro() diff --git a/cmake/target_source_properties.cmake b/cmake/target_source_properties.cmake new file mode 100644 index 0000000000..64bd47379b --- /dev/null +++ b/cmake/target_source_properties.cmake @@ -0,0 +1,142 @@ +# +# These two functions together allow greater control of propagating flags within +# a target on a per-source basis with the ability to "inherit" those properties +# from the target if not set. This allows a target to defing its own flags, but +# then if a file needs different settings those can be directly overridden without +# relying on compiler-specific flag order precedence. Additionally this allows a +# project to organize grouping of flags within a target +# +# Note that for compile defines on source files they are not used in the autogen +# dependency scanning. See : +# https://gitlab.kitware.com/cmake/cmake/-/issues/22519 +# and +# https://gitlab.kitware.com/cmake/cmake/-/blob/master/Source/cmDependsFortran.cxx#L84 +# functions cmDependsFortran::cmDependsFortran() and cmDependsFortran::WriteDependencies() +# +# The solution is to either use Ninja or preprocess the files (what Ninja internally does) +# This is probably the way to go as well since CMake native preprocessor directive +# parsing is... subpar and simplified : +# https://gitlab.kitware.com/cmake/cmake/-/issues/17398 +# +# Alternatively, set critical flags at the target level +# + + + +# +# A simple function to create properties for targets and sources quickly +# +function( define_target_source_properties ) + set( options ) + set( oneValueArgs ) + set( multiValueArgs PROPERTIES ) + + cmake_parse_arguments( + FUNC_PROP + "${options}" "${oneValueArgs}" "${multiValueArgs}" + ${ARGN} + ) + + foreach( PROPERTY ${FUNC_PROP_PROPERTIES} ) + define_property( + SOURCE + PROPERTY ${PROPERTY} + # INHERITED # they will be "inherited" via target to source + ) + + define_property( + TARGET + PROPERTY ${PROPERTY} + # INHERITED # they will be "inherited" via target to source + ) + endforeach() +endfunction() + + +# +# The bulk of the functionality exists in this function. It will loop over each +# provided target, gathering sources and their respective properties listed, using +# the target's property if not defined for this source else nothing, and finally +# applies it to that source. +# +function( apply_target_source_properties ) + set( options DISCARD_PREVIOUS DEBUG ) + set( oneValueArgs AS_PROPERTY ) + set( multiValueArgs TARGETS PROPERTIES ) + + cmake_parse_arguments( + FUNC_PROP + "${options}" "${oneValueArgs}" "${multiValueArgs}" + ${ARGN} + ) + + foreach( TARGET ${FUNC_PROP_TARGETS} ) + # get target sources + get_target_property( TARGET_SOURCES ${TARGET} SOURCES ) + + # get default "inherited" value from target + foreach( PROPERTY ${FUNC_PROP_PROPERTIES} ) + get_target_property( TARGET_PROPERTY_${PROPERTY} ${TARGET} ${PROPERTY} ) + if ( "${TARGET_PROPERTY_${PROPERTY}}" STREQUAL "TARGET_PROPERTY_${PROPERTY}-NOTFOUND" ) + # unset it + set( TARGET_PROPERTY_${PROPERTY} ) + endif() + endforeach() + + foreach( SOURCE ${TARGET_SOURCES} ) + + # We need to accumulate properties since a call to set property will + # override what was there before + set( SOURCE_PROPERTY_ALL ) + + foreach( PROPERTY ${FUNC_PROP_PROPERTIES} ) + # first try source + get_source_file_property( + SOURCE_PROPERTY_${PROPERTY} + ${SOURCE} + TARGET_DIRECTORY ${TARGET} + ${PROPERTY} + ) + if ( "${SOURCE_PROPERTY_${PROPERTY}}" STREQUAL "NOTFOUND" ) + # use target + set( SOURCE_PROPERTY_${PROPERTY} ${TARGET_PROPERTY_${PROPERTY}} ) + endif() + + # Now apply these as prop + if ( NOT "${SOURCE_PROPERTY_${PROPERTY}}" STREQUAL "" ) + if ( ${FUNC_PROP_DEBUG} ) + message( STATUS "DEBUG : Adding '${SOURCE_PROPERTY_${PROPERTY}}' as SOURCE_PROPERTY_${PROPERTY}") + endif() + list( APPEND SOURCE_PROPERTY_ALL ${SOURCE_PROPERTY_${PROPERTY}} ) + endif() + endforeach() # properties + + # Apply properties to source + if ( NOT "${SOURCE_PROPERTY_ALL}" STREQUAL "" ) + if ( NOT ${FUNC_PROP_DISCARD_PREVIOUS} ) + # get old value and append + get_source_file_property( + SOURCE_PROPERTY_ORIG + ${SOURCE} + TARGET_DIRECTORY ${TARGET} + ${FUNC_PROP_AS_PROPERTY} + ) + if ( "${SOURCE_PROPERTY_ORIG}" STREQUAL "NOTFOUND" ) + set( SOURCE_PROPERTY_ORIG ) + endif() + endif() + + if ( ${FUNC_PROP_DEBUG} ) + message( STATUS "DEBUG : ${FUNC_PROP_AS_PROPERTY} being set to '${SOURCE_PROPERTY_ORIG} ${SOURCE_PROPERTY_ALL}'") + endif() + + set_source_files_properties( + ${SOURCE} + TARGET_DIRECTORY ${TARGET} + PROPERTIES + ${FUNC_PROP_AS_PROPERTY} "${SOURCE_PROPERTY_ORIG};${SOURCE_PROPERTY_ALL}" + ) + endif() + endforeach() # sources + endforeach() # targets +endfunction() diff --git a/cmake/template/arch_config.cmake b/cmake/template/arch_config.cmake index 42cba60287..0a655a4e32 100644 --- a/cmake/template/arch_config.cmake +++ b/cmake/template/arch_config.cmake @@ -15,15 +15,16 @@ set( CMAKE_Fortran_FLAGS_INIT "{SFC_FLAGS} {FCBASEOPTS} {BYTESWAPIO}" ) set( CMAKE_C_FLAGS_INIT "{SCC_FLAGS} {CFLAGS_LOCAL}" ) # https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_FLAGS_CONFIG_INIT.html -set( CMAKE_Fortran_FLAGS_DEBUG_INIT "{FCDEBUG}" ) -set( CMAKE_Fortran_FLAGS_RELEASE_INIT "" ) -set( CMAKE_C_FLAGS_DEBUG_INIT "" ) -set( CMAKE_C_FLAGS_RELEASE_INIT "" ) +set( CMAKE_Fortran_FLAGS_Debug_INIT "{FCDEBUG}" ) +set( CMAKE_Fortran_FLAGS_Release_INIT "" ) +set( CMAKE_C_FLAGS_Debug_INIT "" ) +set( CMAKE_C_FLAGS_Release_INIT "" ) # Project specifics now -set( WRF_MPI_Fortran_FLAGS "{DM_FC_FLAGS}" ) -set( WRF_MPI_C_FLAGS "{DM_CC_FLAGS}" ) -set( WRF_ARCH_LOCAL "{ARCH_LOCAL}" ) -set( WRF_M4_FLAGS "{M4_FLAGS}" ) -set( WRF_FCOPTIM "{FCOPTIM}" ) -set( WRF_FCNOOPT "{FCNOOPT}" ) \ No newline at end of file +set( WRF_MPI_Fortran_FLAGS "{DM_FC_FLAGS}" ) +set( WRF_MPI_C_FLAGS "{DM_CC_FLAGS}" ) +set( WRF_ARCH_LOCAL "{ARCH_LOCAL}" ) +set( WRF_M4_FLAGS "{M4_FLAGS}" ) +set( WRF_FCOPTIM "{FCOPTIM}" ) +set( WRF_FCNOOPT "{FCNOOPT}" ) +set( WRF_LINK_FLAGS "{LDFLAGS_LOCAL}" ) diff --git a/frame/CMakeLists.txt b/frame/CMakeLists.txt index 85264c7d5f..4543c4694e 100644 --- a/frame/CMakeLists.txt +++ b/frame/CMakeLists.txt @@ -156,7 +156,7 @@ target_sources( set_source_files_properties( ${nl_dyn_source} PROPERTIES - COMPILE_FLAGS + COMPILE_OPTIONS_OPTIMIZATION $<$:${WRF_FCNOOPT}> ) From 788c882029a43dc7d53cf42dbb6aa3e20bcf8c74 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Wed, 16 Oct 2024 08:39:49 -0700 Subject: [PATCH 27/30] Compilation Tests for ARW (#2115) TYPE: enhancement KEYWORDS: testing, compilation SOURCE: internal DESCRIPTION OF CHANGES: Problem: The testing frameworks (github actions nor Jenkins) don't run a wide array of compilation tests per compiler family commonly available. Solution: Use the github actions / hpc-workflows testing framework to run serial, sm, dm, and dm+sm for GNU, Intel Classic & OneAPI, and PGI/nvhpc on Derecho. Tests will run collectively in one node in parallel to maximize core hour allocations. TESTS CONDUCTED: 1. Tests can be run locally on Derecho by using the .ci/hpc-workflows runner.py to launch all tests as separate jobs, but this is inefficient and not exactly as the tests run. To emulate the tests as done in the github actions workflow use the additional `args` set in the respective testSet of .github/workflows/.ci.yml and provide alternate directories from which to compile. This can be facilitated via : * clone WRF branch that features these changes into `` * list names of tests you want space-delimited in variable, e.g. `export tests="make-gnu-serial make-gnu-dm"` * create copy test dirs with something like `printf "%s\n" $tests | xargs -i -P 4 cp -Rp {}` * Create alt dirs run locations from test definitions. In this case one directory up from each, this can be done with `export altDirs=$( printf "../%s/.ci " $tests )` (note the extra space!) * launch tests using runner and join args along with `-alt $altDirs` * Final command should look like ``` /.ci/hpc-workflows/.ci/runner.py /.ci/wrf_compilation_tests-make.json -t $tests -a -p -tp 1 -j='{"node_select":{"-l ":{"select":1}}}' -jn -alt $altDirs ``` --- .ci/hpc-workflows | 2 +- .ci/wrf_compilation_tests-make.json | 92 ++++++++++++++++++----------- .github/workflows/ci.yml | 18 +++++- .github/workflows/test_workflow.yml | 20 ++++--- 4 files changed, 86 insertions(+), 46 deletions(-) diff --git a/.ci/hpc-workflows b/.ci/hpc-workflows index ba8393447c..dfc8e6d823 160000 --- a/.ci/hpc-workflows +++ b/.ci/hpc-workflows @@ -1 +1 @@ -Subproject commit ba8393447c8a2cef23952c01425154ceb34d64e4 +Subproject commit dfc8e6d823b80497ea41bab94e1fdf3f4594ad18 diff --git a/.ci/wrf_compilation_tests-make.json b/.ci/wrf_compilation_tests-make.json index b80a2b8b53..e261afbf7e 100644 --- a/.ci/wrf_compilation_tests-make.json +++ b/.ci/wrf_compilation_tests-make.json @@ -17,53 +17,75 @@ "queue" : "main", "hpc_arguments" : { - "node_select" : { "-l " : { "select" : 1, "ncpus" : 16 } }, + "node_select" : { "-l " : { "select" : 1, "ncpus" : 8 } }, "priority" : { "-l " : { "job_priority" : "economy" } } }, "arguments" : { - "base_env_numprocs" : [ "-e", "NUM_PROCS=16" ], + "base_env_numprocs" : [ "-e", "NUM_PROCS=8" ], "very_last_modules" : [ "netcdf" ], ".*gnu.*::test_modules" : [ "gcc" ], - ".*intel(?!-llvm).*::test_modules" : [ "intel-classic" ], + ".*intel-classic.*::test_modules" : [ "intel-classic" ], ".*intel-llvm.*::test_modules" : [ "intel-oneapi" ], ".*pgi.*::test_modules" : [ "nvhpc" ], ".*dm.*::test_mpi_module" : [ "cray-mpich" ] } } }, - "make-gnu" : - { - "steps" : - { - "serial" : - { - "command" : ".ci/tests/build.sh", - "arguments" : [ "-c", "32" ] - }, - "sm" : - { - "command" : ".ci/tests/build.sh", - "arguments" : [ "-c", "33" ], - "dependencies" : { "serial" : "afterany" } - } - } + + "make-gnu-serial" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "32" ] } } }, + "make-gnu-sm" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "33" ] } } }, + "make-gnu-dm" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "34" ] } } }, + "make-gnu-dm+sm" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "35" ] } } }, + + "make-intel-classic-serial" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "13" ] } } }, - "make-gnu-mpi" : - { - "steps" : - { - "dm" : - { - "command" : ".ci/tests/build.sh", - "arguments" : [ "-c", "34" ] - }, - "dm+sm" : - { - "command" : ".ci/tests/build.sh", - "arguments" : [ "-c", "35" ], - "dependencies" : { "dm" : "afterany" } - } - } + "make-intel-classic-sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "14" ] } } + }, + "make-intel-classic-dm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "15" ] } } + }, + "make-intel-classic-dm+sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "16" ] } } + }, + + "make-intel-llvm-serial" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "76" ] } } + }, + "make-intel-llvm-sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "77" ] } } + }, + "make-intel-llvm-dm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "78" ] } } + }, + "make-intel-llvm-dm+sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "79" ] } } + }, + + "make-pgi-serial" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "1" ] } } + }, + "make-pgi-sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "2" ] } } + }, + "make-pgi-dm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "3" ] } } + }, + "make-pgi-dm+sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "4" ] } } } } diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec396e2ce2..087f9eba34 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,8 +49,22 @@ jobs: tpool : 1 mkdirs : true tests : - - make-gnu - - make-gnu-mpi + - make-gnu-serial + - make-gnu-sm + - make-gnu-dm + - make-gnu-dm+sm + - make-intel-classic-serial + - make-intel-classic-sm + - make-intel-classic-dm + - make-intel-classic-dm+sm + - make-intel-llvm-serial + - make-intel-llvm-sm + - make-intel-llvm-dm + - make-intel-llvm-dm+sm + - make-pgi-serial + - make-pgi-sm + - make-pgi-dm + - make-pgi-dm+sm # add new compilation tests here uses : ./.github/workflows/test_workflow.yml diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml index abcb901c0a..e80d6f3392 100644 --- a/.github/workflows/test_workflow.yml +++ b/.github/workflows/test_workflow.yml @@ -69,12 +69,15 @@ jobs: if : ${{ inputs.mkdirs }} id : cpTestDirs run : | - for testDir in ${{ join( fromJson( inputs.tests ), ' ' ) }}; do - echo "Creating duplicate directory for $testDir" - # Remove if it exists to get a fresh start - rm -rf $testDir - cp -Rp main/ $testDir - done + # Limit parallel ops to 4 at a time to not thrash login nodes + + # Remove if it exists to get a fresh start + echo "Cleaning old test directories..." + tests="${{ join( fromJson( inputs.tests ), ' ' ) }}" + time printf "%s\n" $tests | xargs -i -P 4 rm -rf {} + + echo "Creating duplicate directory for $tests" + time printf "%s\n" $tests | xargs -i -P 4 cp -Rp main/ {} - name: Test ${{ inputs.name }} id : runTest @@ -87,6 +90,7 @@ jobs: -t ${{ join( fromJson( inputs.tests ), ' ' ) }} \ -a "${{ inputs.account }}" \ -p ${{ inputs.pool}} -tp ${{ inputs.tpool }} \ + -jn ${{ github.event_name == 'push' && github.ref_name || github.event.number }}-${{ inputs.id }} \ ${{ inputs.args }} $ALT_DIRS @@ -129,8 +133,8 @@ jobs: # *documented* functionality doesn't work as expected. Wow, bravo # can't use ${{ env. }} as somehow this combination of matrix->reusable workflow->call step is too complex # and expands to nothing - name: ${{ github.event_name == 'push' && 'master' || github.event.number }}-${{ inputs.id }}_logfiles - path: ${{ inputs.archive }}/${{ github.event_name == 'push' && 'master' || github.event.number }}/${{ inputs.id }}/ + name: ${{ github.event_name == 'push' && github.ref_name || github.event.number }}-${{ inputs.id }}_logfiles + path: ${{ inputs.archive }}/${{ github.event_name == 'push' && github.ref_name || github.event.number }}/${{ inputs.id }}/ # As noted in ci.yml, this will need to be moved to a separate workflow with pull_request_target # and strictly controlled usage of the GH token From fa023f284f0dad1e09e0899bfe3838c38d4dfb2a Mon Sep 17 00:00:00 2001 From: joshi994 <160153650+joshi994@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:47:32 -0400 Subject: [PATCH 28/30] Shortwave radiation balance at the wall missing reflected direct and diffuse radiation reflection from the wall terms. (#2101) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TYPE: Bug fix KEYWORDS: Shortwave radiation balance, reflected radiation, shadowing effect, single-layer urban canopy model (SLUCM), direct and diffuse radiation. SOURCE: Parag Joshi, Katia Lamer (Brookhaven National Laboratory) DESCRIPTION OF CHANGES:
 Problem:
The single-layer urban canopy model missed a couple of terms that represent the direct and diffuse radiation reaching at a wall reflected from the other wall. It leads to an inaccurate calculation of the shortwave radiation at the walls. Solution:
Mathematical formulation by Kusaka et. al. (2001) was followed to verify the equations used in the SLUCM module in WRF. The equations are corrected. More details can be found in this [attachment](https://github.com/user-attachments/files/16800936/Shortwave.radiation_Single.Layer.pdf). LIST OF MODIFIED FILES: module_sf_urban.F TESTS CONDUCTED: 1. It compiles. 2. It passes the Jenkins tests. RELEASE NOTE: This PR corrects the shortwave radiation balance at the wall, particularly the reflected direct and diffuse radiation reaching the wall which leads to underestimation of SW radiation at the wall. --- phys/module_sf_urban.F | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/phys/module_sf_urban.F b/phys/module_sf_urban.F index 42806772c3..048d27dcac 100644 --- a/phys/module_sf_urban.F +++ b/phys/module_sf_urban.F @@ -915,7 +915,7 @@ SUBROUTINE urban(LSOLAR, & ! L SG1=SX*VFGS*(1.-ALBG) SB1=SX*VFWS*(1.-ALBB) SG2=SB1*ALBB/(1.-ALBB)*VFGW*(1.-ALBG) - SB2=SG1*ALBG/(1.-ALBG)*VFWG*(1.-ALBB) + SB2=SG1*ALBG/(1.-ALBG)*VFWG*(1.-ALBB) + SB1*ALBB*VFWW ELSE ! shadow effects model @@ -961,7 +961,7 @@ SUBROUTINE urban(LSOLAR, & ! L SG1=SD*(RW-SLX)/RW*(1.-ALBG)+SQ*VFGS*(1.-ALBG) SB1=SD*SLX/W*(1.-ALBB)+SQ*VFWS*(1.-ALBB) SG2=SB1*ALBB/(1.-ALBB)*VFGW*(1.-ALBG) - SB2=SG1*ALBG/(1.-ALBG)*VFWG*(1.-ALBB) + SB2=SG1*ALBG/(1.-ALBG)*VFWG*(1.-ALBB) + SB1*ALBB*VFWW END IF From b1e1258b0a3fbbb8c426ecd32626ecf790f7c853 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:08:08 -0700 Subject: [PATCH 29/30] CMake netCDF Compatibility with WPS (#2121) TYPE: bug fix KEYWORDS: compilation, cmake, netcdf SOURCE: internal DESCRIPTION OF CHANGES: Problem: PR #2054 changes the netCDF target linking for WRF targets generated through CMake. When used with any other software relying on the package config generated from `cmake/template/WRFConfig.cmake`, there must be a way to find non-CMake supported packages like netCDF. WRF provides these find modules in `/share` but does not use them within its own package config. WPS also has a FindnetCDF.cmake and FindnetCDF-Fortran.cmake, but these use the classic variable approach instead of an import target. Thus, when building WPS referencing WRF with the changes of #2054, it will use the WPS netCDF find modules instead of WRF's. This will result in an error like: ``` CMake Error at /home/aislas/wrf-model/wrf/install/lib/cmake/WRF/WRFTargets.cmake:111 (set_target_properties): The link interface of target "WRF::io_netcdf" contains: netCDF::netcdff ``` Solution: Adjust the generated package config file to use WRF's find modules and give them precedence over WPS for WRF's imported targets specifically. Additionally, move imported target inclusion to after dependencies have been resolved so that WRF package finding precedence is used. TESTS CONDUCTED: 1. Tested with gcc/OpenMPI to build WRF and WPS (v4.6.0) together --- cmake/template/WRFConfig.cmake.in | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmake/template/WRFConfig.cmake.in b/cmake/template/WRFConfig.cmake.in index f896e0f420..7bd1e40223 100644 --- a/cmake/template/WRFConfig.cmake.in +++ b/cmake/template/WRFConfig.cmake.in @@ -2,8 +2,6 @@ @PACKAGE_INIT@ -include( "${CMAKE_CURRENT_LIST_DIR}/@EXPORT_NAME@Targets.cmake" ) - set( WRF_VERSION @PROJECT_VERSION@ ) # Options WRF was built with @@ -37,6 +35,7 @@ set( WRF_BUILD_SBM_FAST @BUILD_SBM_FAST@ ) set( WRF_SHOW_ALL_VARS_USED @SHOW_ALL_VARS_USED@ ) set( WRF_WRFIO_NCD_NO_LARGE_FILE_SUPPORT @WRFIO_NCD_NO_LARGE_FILE_SUPPORT@ ) +list( PREPEND CMAKE_MODULE_PATH @CMAKE_INSTALL_PREFIX@/share/ ) if ( ${WRF_USE_MPI} ) find_package( MPI REQUIRED COMPONENTS Fortran C ) @@ -46,9 +45,14 @@ if ( ${WRF_USE_OPENMP} ) find_package( OpenMP REQUIRED COMPONENTS Fortran C ) endif() -find_package( netCDF REQUIRED ) +find_package( netCDF REQUIRED ) +find_package( netCDF-Fortran REQUIRED ) # Attempt to find zlib packaged with netcdf first set( ZLIB_ROOT ${netCDF_PREFIX} ) find_package( ZLIB REQUIRED ) -check_required_components( "@EXPORT_NAME@_Core" ) \ No newline at end of file +list( POP_FRONT CMAKE_MODULE_PATH ) + +include( "${CMAKE_CURRENT_LIST_DIR}/@EXPORT_NAME@Targets.cmake" ) + +check_required_components( "@EXPORT_NAME@_Core" ) From c21d571d625286369212424810af300ca27495c6 Mon Sep 17 00:00:00 2001 From: Anthony Islas <128631809+islas@users.noreply.github.com> Date: Wed, 16 Oct 2024 13:43:45 -0700 Subject: [PATCH 30/30] Update version info for release v4.6.1 (#2122) TYPE: text only KEYWORDS: v4.6.1, release, version_decl, README SOURCE: internal DESCRIPTION OF CHANGES: Updated the top-level README and inc/version_decl files to reflect V4.6.1, in preparation for the v4.6.1 release LIST OF MODIFIED FILES: M README M inc/version_decl TESTS CONDUCTED: No tests necessary - text only --- README | 2 +- inc/version_decl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README b/README index a2135597d7..64ceeda6b4 100644 --- a/README +++ b/README @@ -1,4 +1,4 @@ -WRF Model Version 4.6.0 +WRF Model Version 4.6.1 https://www2.mmm.ucar.edu/wrf/users/ diff --git a/inc/version_decl b/inc/version_decl index 6cfe90eaba..20fb319054 100644 --- a/inc/version_decl +++ b/inc/version_decl @@ -1 +1 @@ - CHARACTER (LEN=*), PARAMETER :: release_version = 'V4.6.0' + CHARACTER (LEN=*), PARAMETER :: release_version = 'V4.6.1'