! bn_aggregate_t.jnl
! 9/12/2015 *sh* tweak to put all test files into "tmp" subdirectory
! 1/2016 *sh* added implicit naming of aggregation from script filename
! 2/2016      and auto-sorting of member files
! 1/2017 *acm* changes in output due to backing off auto-detection of truemonth axes, tkt 2497

spawn "mkdir -p tmp"     ! tmp directory t store files

! *** create test files for T aggregation ***

! --- files with regularly-spaced time points ---
define axis/t=1-jan-1980:1-jan-1990:1/unit=days tday
let myvar = SIN(T[gt=tday]/10) + L[gt=tday]/10

! also define a time intependent field to be saved as the second variable
! but only in the first data set
let mymap = I[x=0E:0E:5]+ 10*J[Y=80s:80n:5]

save/clobber/file="tmp/tagg_reg_1.nc" myvar[l=1:10], mymap
save/clobber/file="tmp/tagg_reg_2.nc" myvar[l=11:20]
save/clobber/file="tmp/tagg_reg_3.nc" myvar[l=21:30]
save/clobber/file="tmp/tagg_reg_4.nc" myvar[l=31:40]
save/clobber/file="tmp/tagg_reg_5.nc" myvar[l=41:50]
save/clobber/file="tmp/tagg_reg_6.nc" myvar[l=51:60]
save/clobber/file="tmp/tagg_reg_7.nc" myvar[l=61:70]
save/clobber/file="tmp/tagg_reg_8.nc" myvar[l=71:80]
save/clobber/file="tmp/tagg_reg_9.nc" myvar[l=81:90]
save/clobber/file="tmp/tagg_reg_10.nc" myvar[l=91:100]

! Add some global attributes which will propagate through aggregations: 
!  TSERIES inherits global atts from the 1st member
!  ENSEMBLE aggregations promote global attrs to a variable on the E axis

let investigator = "Ferret developers"
let description = "Example for time-aggregation working with global attributes"
save/asglobal/append/file="tmp/tagg_reg_1.nc" investigator, description

! save a duplicate of the first file
save/clobber/file="tmp/tagg_reg_1_duplicate.nc" myvar[l=1:10], mymap

! save a file with no time-dependent variables
save/clobber/file="tmp/tagg_reg_no_T.nc"  mymap

! make a single file that is "too long": 15 time steps instead of 10
! so that when used in a time series it creates reverse-ordered points
define axis/t=1-jan-1980:1-jan-1990:1/unit=days tday
let myvar = SIN(T[gt=tday]/10) + L[gt=tday]/10
save/clobber/file="tmp/tagg_reg_2_overlarge.nc" myvar[l=11:25]
cancel axis tday

! --- files with irregularly-spaced time points ---
define axis/T0=1-jan-1980/unit=days tday_ir = L[l=1:200]^1.01

let myvar = SIN(T[gt=tday_ir]/10) + L[gt=tday_ir]/10
!plot myvar[l=1:100]

save/clobber/file="tmp/tagg_irreg_1.nc" myvar[l=1:10]
save/clobber/file="tmp/tagg_irreg_2.nc" myvar[l=11:20]
save/clobber/file="tmp/tagg_irreg_3.nc" myvar[l=21:30]
save/clobber/file="tmp/tagg_irreg_4.nc" myvar[l=31:40]
save/clobber/file="tmp/tagg_irreg_5.nc" myvar[l=41:50]
save/clobber/file="tmp/tagg_irreg_6.nc" myvar[l=51:60]
save/clobber/file="tmp/tagg_irreg_7.nc" myvar[l=61:70]
save/clobber/file="tmp/tagg_irreg_8.nc" myvar[l=71:80]
save/clobber/file="tmp/tagg_irreg_9.nc" myvar[l=81:90]
save/clobber/file="tmp/tagg_irreg_10.nc" myvar[l=91:100]

let investigator = "Ferret developers"
let description = "Example for time-aggregation working with global attributes"

save/asglobal/append/file="tmp/tagg_irreg_1.nc" investigator
save/asglobal/append/file="tmp/tagg_irreg_1.nc" description

! create a regular time axis (lacking cell bounds) in the midst of the sequence
define axis/T0=1-jan-1980/unit=days tday_ir = L[l=11:20]
save/clobber/file="tmp/tagg_irreg_2_reg.nc" myvar
cancel axis tday_ir


! --- files in which the T0 origin varies ---
! create a sequence of files with differing T0 dates, but the very same
! timestep values (1, 2, 3, 4)

let tt = t[gt=tday_tvar]

repeat/i=1:5 (\
define symbol file `i`;\
define symbol start_day `(i-1)*4+1`;\
define axis/T0=($start_day)-jan-1980/unit=days tday_Tvar = L[l=1:4];\
let myvar0 = TAX_JDAY(L[l=1:4], tt);\
let myvar = myvar0[gt=tday_Tvar@asn];\
save/clobber/file="tmp/tagg_T0varying_($file).nc" myvar[l=1:4])

! --- files with T0 varying and date gaps between files ---
let myvar = L[gt=tday_Tvar]

repeat/i=1:10 (\
define symbol file `i`;\
define symbol year `i+1980`;\
define axis/T0=1-jan-($year)/unit=days tday_Tvar = L[l=1:10];\
save/clobber/file="tmp/tagg_disjoint_T0varying_($file).nc" myvar[l=1:10])

! alter first and third files to have only single time step
define axis/T0=1-jan-1981/unit=days tday_Tvar = L[l=1:10]
save/clobber/file="tmp/tagg_disjoint_T0varying_1.nc" myvar[l=1:1]
define axis/T0=1-jan-1983/unit=days tday_Tvar = L[l=1:10]
save/clobber/file="tmp/tagg_disjoint_T0varying_3.nc" myvar[l=1:1]

! make an alternative file 4 that has a different name for the time axis
define axis/T0=1-jan-1984/unit=days tday_WRONG = L[l=1:10]
let myvar = L[gt=tday_WRONG]
save/clobber/file="tmp/tagg_disjoint_T0varying_4_wrong.nc" myvar[l=1:1]
cancel axis tday_Tvar
cancel axis tday_WRONG

! --- files each containing only a single time step ---
define axis/t=1-jan-1980:1-jan-1990:1/unit=days tday

let myvar = SIN(T[gt=tday]/10) + L[gt=tday]/10

! also define a time intependent field to be saved as the second variable
! but only in the first data set
let mymap = I[x=0E:0E:5]+ 10*J[Y=80s:80n:5]

!save/rigid/keep_axisname/file=...

save/clobber/file="tmp/tagg_single_1.nc" myvar[l=1], mymap
save/clobber/file="tmp/tagg_single_2.nc" myvar[l=2]
save/clobber/file="tmp/tagg_single_3.nc" myvar[l=3]
save/clobber/file="tmp/tagg_single_4.nc" myvar[l=4]
save/clobber/file="tmp/tagg_single_5.nc" myvar[l=5]
save/clobber/file="tmp/tagg_single_6.nc" myvar[l=6]

cancel axis tday

cancel var/all


! *****************************************
! *** test the aggregation capabilities ***
! --- regularly-spaced axes ---
! if the first file of the aggregation is already open
! it must not get disturbed by setting up the aggregation
use tmp/tagg_reg_1.nc

! single dataset aggregation
tseries myagg1 =  tmp/tagg_reg_1.nc
show data myagg1
show grid myvar

! double file aggregation
define data/agg/t myagg2 = tmp/tagg_reg_1.nc, tmp/tagg_reg_2.nc
show grid myvar
stat myvar

! show that aggregate name is case-sensitive
define data/agg/t myAgg2 = tmp/tagg_reg_1.nc, tmp/tagg_reg_2.nc
show data myAgg2

! *sh* 1/28/2016 addition
! same aggregation inheriting its name from the parent script
define data/agg/t tmp/tagg_reg_1.nc, tmp/tagg_reg_2.nc
show data/brief

! define a long aggregation
define data/agg/t myagg8 = tmp/tagg_reg_1.nc, tmp/tagg_reg_2.nc,tmp/tagg_reg_3.nc, tmp/tagg_reg_4.nc,tmp/tagg_reg_5.nc, tmp/tagg_reg_6.nc,tmp/tagg_reg_7.nc, tmp/tagg_reg_8.nc
show data/full  ! all 4 datasets
show grid myvar
plot/line/sym myvar

! read a time-independent field from the first file of the aggregation
stat mymap

! 2/2016 -- reorder deliberately out of order regular files
define data/agg/t myagg8_disordered1 = tmp/tagg_reg_2.nc, tmp/tagg_reg_1.nc,tmp/tagg_reg_3.nc, tmp/tagg_reg_4.nc,tmp/tagg_reg_5.nc, tmp/tagg_reg_6.nc,tmp/tagg_reg_7.nc, tmp/tagg_reg_8.nc
define data/agg/t myagg8_disordered2 = tmp/tagg_reg_1.nc, tmp/tagg_reg_2.nc,tmp/tagg_reg_3.nc, tmp/tagg_reg_4.nc,tmp/tagg_reg_5.nc, tmp/tagg_reg_6.nc,tmp/tagg_reg_8.nc, tmp/tagg_reg_7.nc
show data/files/brief myagg8_disordered1
show data/files/brief myagg8_disordered2
stat/brief T[g=myvar[d=myagg8]] - T[g=myvar[d=myagg8_disordered1]]
stat/brief TBOXLO[g=myvar[d=myagg8]] - TBOXLO[g=myvar[d=myagg8_disordered1]]
stat/brief TBOXHI[g=myvar[d=myagg8]] - TBOXHI[g=myvar[d=myagg8_disordered1]]
cancel data/all

! deliberate errors
set mode ignore
! try to define the same aggregation again
define data/agg/t myagg2 = tmp/tagg_reg_1.nc, tmp/tagg_reg_2.nc

! try to aggregate with the same file duplicated
define data/agg/t myagg2 = tmp/tagg_reg_1.nc, tmp/tagg_reg_2.nc, tmp/tagg_reg_2.nc

! try to aggregate a missing file as first file
define data/agg/t myaggbad = tmp/tagg_no_exist.nc, tmp/tagg_reg_2.nc

! try to aggregate a missing file as subsequent file
define data/agg/t myaggbad = tmp/tagg_reg_1.nc, tmp/tagg_no_exist.nc

! try to define an aggregation where there is no time axis in the first file
tseries myTindepagg = tmp/tagg_reg_no_T.nc

! erroneous file 2 with 15 time steps, goes beyond start of file 3
define data/agg/t myagg3 = tmp/tagg_reg_1.nc, tmp/tagg_reg_2_overlarge.nc,tmp/tagg_reg_3.nc

! Try to use dataset names not file names. This is not allowed for Tseries aggregations.
use tmp/tagg_reg_1.nc, tmp/tagg_reg_2.nc
define data/agg/t myagg2num = 1,2

set mode/last ignore
canc data/all

! --- irregularly-spaced axes ---
! aggregation of 2 irregular datasets
 define data/agg/t myagg2 = tmp/tagg_irreg_1.nc, tmp/tagg_irreg_2.nc

! aggregate a longer collection where all are irregular
define data/agg/t myagg8 = tmp/tagg_irreg_1.nc, tmp/tagg_irreg_2.nc,tmp/tagg_irreg_3.nc, tmp/tagg_irreg_4.nc,tmp/tagg_irreg_5.nc, tmp/tagg_irreg_6.nc,tmp/tagg_irreg_7.nc, tmp/tagg_irreg_8.nc
  show data/full
  show data myagg2
  show grid/t myvar[d=1]
  show grid myvar[d=2]
  plot/line/sym myvar[d=2],myvar[d=1] 

! aggregate collection that mixes regular (file 2) with irregular
define data/agg/t myagg4 = tmp/tagg_irreg_1.nc, tmp/tagg_irreg_2_reg.nc,tmp/tagg_irreg_3.nc, tmp/tagg_irreg_4.nc
  show grid/t myvar[d=myagg4]

! deliberate error -- aggregated file out of order
set mode ignore
define data/agg/t myagg3 = tmp/tagg_irreg_1.nc, tmp/tagg_irreg_3.nc,tmp/tagg_irreg_2.nc
set mode/last ignore
canc data/all

! --- aggregation from T0-varying files ---
! aggregate two
  define data/agg/t myagg2 = tmp/tagg_T0varying_1.nc, tmp/tagg_T0varying_2.nc

! aggregate several
define data/agg/t myagg8 = tmp/tagg_T0varying_1.nc, tmp/tagg_T0varying_2.nc,tmp/tagg_T0varying_3.nc, tmp/tagg_T0varying_4.nc,tmp/tagg_T0varying_5.nc
  show data/full
  show data myagg2
  show grid/t myvar[d=1]
  show grid myvar[d=2]
  plot/line/sym myvar[d=2],myvar[d=1] 

canc data/all

! --- aggregations of disjoint, irregularly-sized, T0-varying files ---
! aggregate two
  define data/agg/t myagg2 = tmp/tagg_disjoint_T0varying_1.nc, tmp/tagg_disjoint_T0varying_2.nc


! aggregate several
define data/agg/t myagg8 = tmp/tagg_disjoint_T0varying_1.nc, tmp/tagg_disjoint_T0varying_2.nc,tmp/tagg_disjoint_T0varying_3.nc, tmp/tagg_disjoint_T0varying_4.nc,tmp/tagg_disjoint_T0varying_5.nc, tmp/tagg_disjoint_T0varying_6.nc,tmp/tagg_disjoint_T0varying_7.nc, tmp/tagg_disjoint_T0varying_8.nc
  show data/full
  show data myagg2
  show grid/t myvar[d=1]
  show grid myvar[d=2]
  plot/line/sym/vlim=0:11 myvar[d=2],myvar[d=1] 

! a few tests of using Ferret string variables to get the names of files
define data/agg/t listAgg1 = {"tmp/tagg_reg_1.nc", "tmp/tagg_reg_2.nc"}
show data/files listAgg1

! define a long aggregation
let file_list = SPAWN("ls -1 tmp/tagg_reg_?.nc") 
define data/agg/t listAgg2 = file_list
show data/files listAgg2

! reorder deliberately out of order file of differing lengths
define data/agg/t myagg8_disordered3 = tmp/tagg_disjoint_T0varying_2.nc, tmp/tagg_disjoint_T0varying_1.nc,tmp/tagg_disjoint_T0varying_5.nc, tmp/tagg_disjoint_T0varying_4.nc,tmp/tagg_disjoint_T0varying_3.nc, tmp/tagg_disjoint_T0varying_8.nc,tmp/tagg_disjoint_T0varying_7.nc, tmp/tagg_disjoint_T0varying_6.nc
show data/files/brief myagg8_disordered3
! ... the same calendar axis, but encoded with the T0 of the first file
show grid/l=1:8 myvar[d=myagg8]
show grid/l=1:8 myvar[d=myagg8_disordered3]

cancel data/all

! deliberate errors
set mode ignore
! deliberate error - try to re-use a name already in use
  define data/agg/t myagg2 = tmp/tagg_disjoint_T0varying_1.nc, tmp/tagg_disjoint_T0varying_2.nc

! deliberate error - the fourth file (*_4_wrong.nc) changes time axis name
define data/agg/t myagg5_bad = tmp/tagg_disjoint_T0varying_1.nc, tmp/tagg_disjoint_T0varying_2.nc,tmp/tagg_disjoint_T0varying_3.nc, tmp/tagg_disjoint_T0varying_4_wrong.nc,tmp/tagg_disjoint_T0varying_5.nc
set mode/last ignore

canc data/all

! --- aggregations of single timestep files ---
! define a regular aggregation
define data/agg/t MYsing_reg_agg6 = tmp/tagg_single_1.nc, tmp/tagg_single_2.nc,tmp/tagg_single_3.nc, tmp/tagg_single_4.nc,tmp/tagg_single_5.nc, tmp/tagg_single_6.nc
show data/full  MYsing_reg_agg6
show grid/t myvar

! read a time-independent field from the first file of the aggregation
stat mymap

! define an irregular aggregation
define data/agg/t MYsing_irreg_agg4 = tmp/tagg_single_1.nc, tmp/tagg_single_2.nc,tmp/tagg_single_4.nc, tmp/tagg_single_5.nc
show data/full MYsing_irreg_agg4
show grid/t myvar

! 2/2016 - reorder deliberately out of order aggregation of irregular times
define data/agg/t MYsing_irreg_agg4_disordered1 = tmp/tagg_single_2.nc, tmp/tagg_single_1.nc,tmp/tagg_single_4.nc, tmp/tagg_single_5.nc
define data/agg/t MYsing_irreg_agg4_disordered2 = tmp/tagg_single_1.nc, tmp/tagg_single_2.nc,tmp/tagg_single_5.nc, tmp/tagg_single_4.nc
show data/files/brief
stat/brief T[g=myvar[d=MYsing_irreg_agg4]] - T[g=myvar[d=MYsing_irreg_agg4_disordered1]]
stat/brief TBOXLO[g=myvar[d=MYsing_irreg_agg4]] - TBOXLO[g=myvar[d=MYsing_irreg_agg4_disordered1]]
stat/brief TBOXHI[g=myvar[d=MYsing_irreg_agg4]] - TBOXHI[g=myvar[d=MYsing_irreg_agg4_disordered1]]
canc data/all

! OPeNDAP test
! (Note: when running pre-true-month Ferret versions, the timestep output from 
! show grid/T below was eliminated by the "clean_draconian" script)
! 1/17/2017 changes due to backing off auto-detection of truemonth axes, tkt 2497

IF `test_opendap("http://ferret.pmel.noaa.gov/pmel/thredds/dodsC/data/ferret/test_coads_monthly_air_2012.nc") EQ 0` THEN
   tseries COADS = http://ferret.pmel.noaa.gov/pmel/thredds/dodsC/data/ferret/test_coads_monthly_air_2012.nc, http://ferret.pmel.noaa.gov/pmel/thredds/dodsC/data/ferret/test_coads_monthly_air_2013.nc, http://ferret.pmel.noaa.gov/pmel/thredds/dodsC/data/ferret/test_coads_monthly_air_2014.nc
show data COADS
show grid/T air
canc data/all
ELSE
   say OPenDAP data not available for testing.. Tests skipped ...
ENDIF

! "USE file.agg" test
! test the hack to XEQ_SET which permits "SET DATA file.agg" to metamorphose
! into "GO file.agg"
spawn echo "define data/agg/t myagg3 = tmp/tagg_reg_1.nc, tmp/tagg_reg_2.nc,tmp/tagg_reg_3.nc" > tmp/good_agg.agg
spawn echo "garbage command syntax" > tmp/bad_agg.agg

! deliberate errorrs
! bad .agg aggregation file
set mode ignore
USE "tmp/bad_agg.agg"

! irregular aggregation with first file duplicated
define data/agg/t MYsing_irreg_agg4 = tmp/tagg_single_1.nc, tmp/tagg_single_1.nc, tmp/tagg_single_2.nc,tmp/tagg_single_4.nc, tmp/tagg_single_5.nc

! irregular aggregation with last file duplicated
define data/agg/t MYsing_irreg_agg4 = tmp/tagg_single_1.nc, tmp/tagg_single_2.nc,tmp/tagg_single_4.nc, tmp/tagg_single_5.nc, tmp/tagg_single_5.nc

set/last mode ignore

! valid syntax to initialize an aggregation
USE "tmp/good_agg.agg"
show data/br
cancel data/all
