-
Notifications
You must be signed in to change notification settings - Fork 9
/
NOTES.txt
413 lines (259 loc) · 17.9 KB
/
NOTES.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
NOTES
## Add python3.8 and modules needed for tools
## need to check what libs we actually want
RUN cd / \
&& wget https://www.python.org/ftp/python/${PYTHON3_VERSION}/Python-${PYTHON3_VERSION}.tgz \
&& tar xvf Python-${PYTHON3_VERSION}.tgz \
&& cd Python-${PYTHON3_VERSION} \
&& ./configure --enable-optimizations \
&& make -j4 \
&& make altinstall \
&& cd / \
&& python3.8 --version \
&& python3.8 -m pip install ssl \
&& python3.8 -m pip install --upgrade pip \
&& python3.8 -m pip install ssl configparser numpy scipy matplotlib cftime pandas netCDF4 xarray scitools3 \
&& cd / \
&& rm -r Python-${PYTHON3_VERSION} \
&& rm Python-${PYTHON3_VERSION}.tgz
Jupyter
https://jupyter-docker-stacks.readthedocs.io/en/latest/
http://127.0.0.1:10000/lab
http://127.0.0.1:10000/lab
make sure you include the web token!!
docker run -p 10000:8888 docker-jupyter-extensible:latest
follow this for building our own custom jupyter-lab environment for running CTSM/ELM containers
and analyzing output
https://herrmann.tech/en/blog/2021/02/08/how-to-build-custom-environment-for-jupyter-in-docker.html
https://jupyter-docker-stacks.readthedocs.io/en/latest/using/selecting.html
https://dev.to/juanbelieni/how-to-run-jupyterlab-on-docker-4n80
https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/Dockerfile
https://faun.pub/set-current-host-user-for-docker-container-4e521cef9ffc
MORE GOOD INFO IN THIS!
https://mybinder.readthedocs.io/en/latest/tutorials/dockerfile.html
ARG NB_USER=jovyan
ARG NB_UID=1000
ENV USER ${NB_USER}
ENV NB_UID ${NB_UID}
ENV HOME /home/${NB_USER}
RUN adduser --disabled-password \
--gecos "Default user" \
--uid ${NB_UID} \
${NB_USER}
SSH keys and builds - need to sort out how to use ssh keys for checking out submodules
https://docs.docker.com/docker-hub/builds/#build-repositories-with-linked-private-submodules
OLMT NOTES
python site_fullrun.py --site US-Dk3 --caseidprefix MYTEST --nyears_ad_spinup 200 --nyears_final_spinup 400 \
--tstep 1 --cpl_bypass --compiler gnu --mpilib openmpi --gswp3 --model_root /data/sserbin/Modeling/FASSt-simulation/E3SM \
--spinup_vars --sitegroup AmeriFlux --no_submit \
--runroot /data2/Model_Output/elm_runs/olmt --machine modex
# need to install ncl in container
conda install -c conda-forge ncl
# on modex
conda create -p /data2/sserbin/conda_envs/olmt_ncl -c conda-forge ncl python=3.9
https://anaconda.org/conda-forge/ncl
# but how to get ncl installed on the container without creating a conda env?
RUN set -ex \
&& conda install --quiet --yes \
'ncl' \
&& conda clean --all -f -y
https://hub.docker.com/r/earthlab/ncl/dockerfile
RUN conda install --yes -c conda-forge ncl
ALSO NEED NCO
conda install -c conda-forge nco
Also need NCAP from NCO!!
docker run -t -i --rm --hostname=docker --user root fasstsimulation/baseos:gcc850ompi316-latest
docker run -t -i --rm --hostname=docker --user $(id -u):$(id -g) \
-v /Volumes/data/Model_Data/cesm_input_datasets:/inputdata \
-v /Volumes/data2/Model_Output:/output fasstsimulation/elm-builds:release-v2.0.0-latest
docker run -t -i --rm --hostname=docker --user modeluser \
-v /Volumes/data/Model_Data/cesm_input_datasets:/inputdata \
-v /Volumes/data2/Model_Output:/output fasstsimulation/elm-builds:release-v2.0.0-latest
# quick and dirty example testing with docker
python site_fullrun.py --site US-Dk3 --caseidprefix MYTEST --nyears_ad_spinup 200 --nyears_final_spinup 400 \
--tstep 1 --cpl_bypass --compiler gnu --mpilib openmpi --gswp3 --model_root /E3SM --spinup_vars --sitegroup AmeriFlux --no_submit \
--ccsm_input /inputdata --caseroot /output/elm/olmt_cases --runroot /output/olmt/testing --np 2 --machine docker
# this sort of works - but still trying to get caseroot to be OUTSIDE of E3SM modelroot
# OH!!! the output sub folder has to exist! otherwise it defaults to model_root/cime/scripts!!
# so either run like below or make some subfolders in the host /output location to map into!!
docker run -t -i --rm --hostname=docker --user $(id -u):$(id -g) \
-v /Volumes/data/Model_Data/cesm_input_datasets:/inputdata \
-v /Volumes/data2/Model_Output:/output fasstsimulation/elm-builds:release-v2.0.0-latest \
bin/sh -c 'cd /tools/OLMT && python site_fullrun.py --site US-Dk3 --sitegroup AmeriFlux --caseidprefix MYTEST \
--nyears_ad_spinup 200 --nyears_final_spinup 400 --tstep 1 --cpl_bypass --compiler gnu --mpilib openmpi --gswp3 \
--model_root /E3SM --caseroot /output --ccsm_input /inputdata --runroot /output \
--spinup_vars --no_submit --np 2 --machine docker'
docker run -t -i --rm --hostname=docker --user $(id -u):$(id -g) \
-v /Volumes/data/Model_Data/cesm_input_datasets:/inputdata \
-v /Users/sserbin/Data/scratch/olmt_testing:/output fasstsimulation/elm-builds:release-v2.0.0-latest
docker run -t -i --rm --hostname=docker --user "$(id -u):$(id -g)" \
-v /Volumes/data/Model_Data/cesm_input_datasets:/inputdata \
-v /Users/sserbin/Data/scratch/olmt_testing:/output fasstsimulation/elm-builds:release-v2.0.0-latest \
bin/sh -c 'cd /tools/OLMT && python site_fullrun.py --site US-Dk3 --sitegroup AmeriFlux --caseidprefix MYTEST \
--nyears_ad_spinup 200 --nyears_final_spinup 400 --tstep 1 --cpl_bypass --compiler gnu --mpilib openmpi --gswp3 \
--model_root /E3SM --caseroot /output --ccsm_input /inputdata --runroot /output \
--spinup_vars --no_submit --np 1 --machine docker'
# why dont I get permission problems as root? what is going on with folder permissions now?
docker run -t -i --rm --hostname=docker --user root \
-v /Volumes/data/Model_Data/cesm_input_datasets:/inputdata \
-v /Users/sserbin/Data/scratch/olmt_testing:/output fasstsimulation/elm-builds:release-v2.0.0-latest \
bin/sh -c 'cd /tools/OLMT && python site_fullrun.py --site US-Dk3 --sitegroup AmeriFlux --caseidprefix MYTEST \
--nyears_ad_spinup 200 --nyears_final_spinup 400 --tstep 1 --cpl_bypass --compiler gnu --mpilib openmpi --gswp3 \
--model_root /E3SM --caseroot /output --ccsm_input /inputdata --runroot /output \
--spinup_vars --no_submit --np 2 --machine docker'
docker run -t -i --hostname=docker --user "$(id -u):$(id -g)" \
-v /Volumes/data/Model_Data/cesm_input_datasets:/inputdata \
-v /Users/sserbin/Data/scratch/olmt_testing:/output \
fasstsimulation/elm-builds:release-v2.0.0-latest \
/bin/sh -c 'cd /output/cime_case_dirs/MYTEST_US-Dk3_ICB1850CNRDCTCBC_ad_spinup/ && ./case.submit'
Dan's NOTES
https://docs.google.com/document/d/1apsf8lyjRqzwnq1jEoV_9NqlIKuctZUexDt0i91uLwo/edit
https://www.stonybrook.edu/commcms/ookami/support/faq/core-thread-control-ookami
https://slurm.schedmd.com/cons_res_share.html
## singularity
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test containers/elm-builds_release-v2.0.0-latest.sif \
/bin/sh -c 'cd /tools/OLMT && ls -l'
#with fakeroot
singularity exec --fakeroot -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ -B ./OLMT/:/tools/OLMT/ containers/elm-builds_release-v2.0.0-latest.sif \
/bin/sh -c 'cd /tools/OLMT && ls -l /output && python site_fullrun.py --site US-Dk3 --sitegroup AmeriFlux --caseidprefix MYTEST \
--nyears_ad_spinup 200 --nyears_final_spinup 400 --tstep 1 --cpl_bypass --compiler gnu --mpilib openmpi --gswp3 \
--model_root /E3SM --caseroot /output --ccsm_input /inputdata --runroot /output \
--spinup_vars --no_submit --np 1 --machine modex'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ -B ./OLMT/:/tools/OLMT/ containers/elm-builds_release-v2.0.0-latest.sif \
/bin/sh -c 'cd /tools/OLMT && ls -l /output && python site_fullrun.py --site US-Dk3 --sitegroup AmeriFlux --caseidprefix MYTEST \
--nyears_ad_spinup 400 --nyears_final_spinup 400 --tstep 1 --spinup_vars --cpl_bypass --compiler gnu --mpilib openmpi --gswp3 \
--model_root /E3SM --caseroot /output --ccsm_input /inputdata --runroot /output --no_submit --np 1 --machine modex'
# using version 1.2
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ -B ./OLMT/:/tools/OLMT/ containers/elm-builds_release-v1.2.0-latest.sif \
/bin/sh -c 'cd /tools/OLMT && ls -l /output && python site_fullrun.py --site US-Dk3 --sitegroup AmeriFlux --caseidprefix MYTEST \
--nyears_ad_spinup 200 --nyears_final_spinup 400 --tstep 1 --cpl_bypass \
--compiler gnu --mpilib openmpi --gswp3 --model_root /E3SM --caseroot /output --ccsm_input /inputdata --runroot /output \
--no_submit --np 1 --machine modex'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ \
containers/elm-builds_release-v1.2.0-latest.sif \
/bin/sh -c 'cd /output/cime_case_dirs/MYTEST_US-Dk3_ICB1850CNRDCTCBC_ad_spinup && ./preview_namelists'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ \
containers/elm-builds_release-v1.2.0-latest.sif \
/bin/sh -c 'cd /output/cime_case_dirs/MYTEST_US-Dk3_ICB1850CNRDCTCBC_ad_spinup && ./case.submit'
# using NGEE Arctic version 1.2
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ -B ./OLMT/:/tools/OLMT/ containers/elm-builds:release-v1.2.0_ngeearctic-latest.sif \
/bin/sh -c 'cd /tools/OLMT && ls -l /output && python site_fullrun.py --site US-Dk3 --sitegroup AmeriFlux --caseidprefix MYTEST \
--nyears_ad_spinup 200 --nyears_final_spinup 400 --tstep 1 --cpl_bypass \
--compiler gnu --mpilib openmpi --gswp3 --model_root /E3SM --caseroot /output --ccsm_input /inputdata --runroot /output \
--no_submit --np 1 --machine modex'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ \
containers/elm-builds:release-v1.2.0_ngeearctic-latest.sif \
/bin/sh -c 'cd /output/cime_case_dirs/MYTEST_US-Dk3_ICB1850CNRDCTCBC_ad_spinup && ./case.submit'
sbatch submit_job.sh cime_case_dirs/MYTEST_US-Dk3_ICB1850CNRDCTCBC_ad_spinup
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ -B ./OLMT/:/tools/OLMT/ containers/elm-builds:release-v1.2.0_ngeearctic-latest.sif \
/bin/sh -c 'cd /tools/OLMT && python adjust_restart.py --rundir /output/MYTEST_US-Dk3_ICB1850CNRDCTCBC_ad_spinup/run/ \
--casename MYTEST_US-Dk3_ICB1850CNRDCTCBC_ad_spinup --restart_year 401 --model_name elm'
sbatch submit_job.sh cime_case_dirs/MYTEST_US-Dk3_ICB1850CNPRDCTCBC
--pio_version 2 "is default"
# multiple cases
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ -B ./OLMT/:/tools/OLMT/ containers/elm-builds_release-v2.0.0-latest.sif \
/bin/sh -c 'cd /tools/OLMT && ls -l /output && python site_fullrun.py --site US-Dk3 --sitegroup AmeriFlux --caseidprefix MYTEST \
--nyears_ad_spinup 200 --nyears_final_spinup 400 --tstep 1 --cpl_bypass \
--compiler gnu --mpilib openmpi --gswp3 --model_root /E3SM --caseroot /output --ccsm_input /inputdata --runroot /output \
--spinup_vars --no_submit --np 1 --machine modex'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ \
containers/elm-builds_release-v2.0.0-latest.sif \
/bin/sh -c 'cd /output/cime_case_dirs/MYTEST_US-Dk3_ICB1850CNRDCTCBC_ad_spinup && ./preview_run'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ \
containers/elm-builds_release-v2.0.0-latest.sif \
/bin/sh -c 'cd /output/cime_case_dirs/MYTEST_US-Dk3_ICB1850CNRDCTCBC_ad_spinup && ./check_input_data'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ \
containers/elm-builds_release-v2.0.0-latest.sif \
/bin/sh -c 'cd /output/cime_case_dirs/MYTEST_US-Dk3_ICB1850CNRDCTCBC_ad_spinup && ./preview_namelists'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ \
containers/elm-builds_release-v2.0.0-latest.sif \
/bin/sh -c 'cd /output/cime_case_dirs/MYTEST_US-Dk3_ICB1850CNRDCTCBC_ad_spinup && ./case.submit'
sbatch submit_job.sh cime_case_dirs/MYTEST_US-Dk3_ICB1850CNRDCTCBC_ad_spinup
### !! this also looks like it could work
### trying again with ELM1.2 instead. still have to use -np 1.
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ -B ./OLMT:/tools/OLMT/ containers/elm-builds_release-v1.2.0-latest.sif \
/bin/sh -c 'cd /tools/OLMT && ls -l /output && python site_fullrun.py --site US-Dk3 --sitegroup AmeriFlux --caseidprefix MYTEST \
--nyears_ad_spinup 200 --nyears_final_spinup 400 --tstep 1 --cpl_bypass --compiler gnu --mpilib openmpi --gswp3 \
--model_root /E3SM --caseroot /output --ccsm_input /inputdata --runroot /output \
--spinup_vars --no_submit --np 1 --machine modex'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B /data2/Model_Output/olmt_test/:/output/ \
containers/elm-builds_release-v1.2.0-latest.sif \
/bin/sh -c 'cd /output/cime_case_dirs/MYTEST_US-Dk3_ICB1850CNRDCTCBC_ad_spinup && ./case.submit'
### Atempting to mimic with our own custom workflow setup. Use the non coupler bypass compset
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B scripts:/scripts -B /data2/Model_Output/elm_runs/single_point:/output \
containers/elm-builds_release-v2.0.0-latest.sif \
/scripts/./create_newcase_elm_1pt_custom_site.sh --case_root=/output \
--site_name=US-xTA --start_year='1901-01-01' --num_years=125 --output_vars=/scripts/output_vars.txt --output_freq=H \
--met_start=1901 --met_end=2014 --compset=IGSWELMBGC --rmachine=modex
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata -B /data2/Model_Output/elm_runs/single_point:/output \
containers/elm-builds_release-v2.0.0-latest.sif /bin/sh -c 'cd /output/US-xTA.ELM.IGSWELMBGC.1652114023 && ./preview_namelists'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata -B /data2/Model_Output/elm_runs/single_point:/output \
containers/elm-builds_release-v2.0.0-latest.sif /bin/sh -c 'cd /output/US-xTA.ELM.IGSWELMBGC.1652114023 && ./case.submit'
# works
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B simulation_containers/scripts/hlm-fates/elm_run_scripts/:/scripts \
-B /data2/Model_Output/elm_runs/single_point:/output \
containers/elm-builds_release-v2.0.0-latest.sif \
/scripts/./create_newcase_elm_1pt_custom_site_spinup.sh --case_root=/output \
--site_name=US-xTA --start_year='0001-01-01' --num_years=200 \
--output_vars=/scripts/output_vars.txt --output_freq=H \
--met_start=1901 --met_end=2014 --compset=IGSWELMBGC --rmachine=modex
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata -B /data2/Model_Output/elm_runs/single_point:/output \
containers/elm-builds_release-v2.0.0-latest.sif /bin/sh -c 'cd /output/US-xTA.ELM.IGSWELMBGC.1652114736.spinup && ./preview_namelists'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata -B /data2/Model_Output/elm_runs/single_point:/output \
containers/elm-builds_release-v2.0.0-latest.sif /bin/sh -c 'cd /output/US-xTA.ELM.IGSWELMBGC.1652115294.spinup && ./case.submit'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B simulation_containers/scripts/hlm-fates/elm_run_scripts/:/scripts \
-B /data2/Model_Output/elm_runs/single_point:/output \
containers/elm-builds_release-v2.0.0-latest.sif \
/scripts/./create_newcase_elm_1pt_custom_site_spinup.sh --case_root=/output \
--site_name=US-xTA --start_year='0001-01-01' --num_years=200 \
--output_vars=/scripts/output_vars.txt --output_freq=H \
--met_start=1901 --met_end=2014 --compset=I1850GSWELMBGC --rmachine=modex
# example batch submit. need to make this flexible
sbatch submit_job.sh US-xTA.ELM.I1850GSWELMBGC.1652115741.spinup
## USING output_vars3.txt
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B simulation_containers/scripts/hlm-fates/elm_run_scripts/:/scripts \
-B /data2/Model_Output/elm_runs/single_point:/output \
containers/elm-builds_release-v2.0.0-latest.sif \
/scripts/./create_newcase_elm_1pt_custom_site_ad_spinup.sh --case_root=/output \
--site_name=US-xTA --start_year='0001-01-01' --num_years=200 \
--output_vars=/scripts/output_vars_3.txt --output_freq=H \
--met_start=1901 --met_end=2014 --compset=I1850GSWELMBGC --rmachine=modex
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata -B /data2/Model_Output/elm_runs/single_point:/output \
containers/elm-builds_release-v2.0.0-latest.sif /bin/sh -c 'cd /output/US-xTA_ELM_IGSWELMBGC_ad_spinup && ./preview_namelists'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata -B /data2/Model_Output/elm_runs/single_point:/output \
containers/elm-builds_release-v2.0.0-latest.sif /bin/sh -c 'cd /output/US-xTA_ELM_IGSWELMBGC_ad_spinup && ./case.submit'
# example batch submit. need to make this flexible
sbatch submit_job.sh US-xTA_ELM_I1850GSWELMBGC_ad_spinup
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata \
-B simulation_containers/scripts/hlm-fates/elm_run_scripts/:/scripts \
-B /data2/Model_Output/elm_runs/single_point:/output \
containers/elm-builds_release-v2.0.0-latest.sif \
/scripts/./create_newcase_elm_1pt_custom_site_final_spinup.sh --case_root=/output \
--site_name=US-xTA --run_type=hybrid --start_year='0201-01-01' --num_years=400 \
--output_vars=/scripts/output_vars_3.txt --output_freq=H \
--met_start=1901 --met_end=2014 --compset=IGSWELMBGC --rmachine=modex
sbatch submit_job.sh US-xTA_ELM_I1850GSWELMBGC_final_spinup
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata -B /data2/Model_Output/elm_runs/single_point:/output \
containers/elm-builds_release-v2.0.0-latest.sif /bin/sh -c 'cd /output/US-xTA_ELM_I1850GSWELMBGC_final_spinup && ./preview_namelists'
singularity exec -B /data/Model_Data/cesm_input_datasets:/inputdata -B /data2/Model_Output/elm_runs/single_point:/output \
containers/elm-builds_release-v2.0.0-latest.sif /bin/sh -c 'cd /output/US-xTA_ELM_I1850GSWELMBGC_final_spinup && ./case.submit'