;**************************************************************
;  Gets the merged sounding data
;  date       yyyymmdd  search string
;  site       arm site
;  fill_flag  yes,no  if you want to fill in the data
;             gaps with missing data arrays
;**************************************************************

pro get_merged_sounding_list,varname,data,site,date,fill_flag,missing

;**************************************************************
;  Inputs can be set here or passed in through the command line
;**************************************************************

;  Site to process
;site='nsa'

;  Date to process
;date='199801'

;  Variables to get
;varname=['base_time','time_offset','height','temp','pressure']

;  Fill in the data gaps with missing data arrays
;fill_flag='no'

;  Missing data value
;missing=-8888

;**************************************************************
;  Establish some constants
;**************************************************************

;  Printing flag.  Set dp='yes' if you want to see print statements
dp='no'

;  Main arm site directory
arm_dir='/uufs/chpc.utah.edu/common/home/mace_grp/data/arm/'+site+'/'

;  Data directory depends on site
if site eq 'sgp' then begin
  datastream='sgp_merged_sounding'
endif else if site eq 'nsa' then begin
  datastream='nsa_merged_sounding'
endif else if site eq 'twpc1' then begin
  datastream='twpc1_merged_sounding'
endif else if site eq 'twpc2' then begin
  datastream='twpc2_merged_sounding'
endif else if site eq 'twpc3' then begin
  datastream='twpc3_merged_sounding'
endif

;  List of 2d variables in the file and their dimensions
var2=[$
'temp',$
'pressure',$
'mxrat',$
'uwind',$
'vwind']
vardim2=make_array(n_elements(var2),/int,value=2)

;  List of 1d variables in the file and their dimensions
var1=[$
'sfc_temp',$
'sfc_pressure',$
'sfc_mxrat',$
'vap',$
'sndg_flag']
vardim1=make_array(n_elements(var1),/int,value=1)

;  Variables that aren't to be combined across files
var0=['height','base_time','time_offset']
vardim0=make_array(n_elements(var0),/int,value=0)

;  Combine these all into one big array
var=[var2,var1,var0]
vardim=[vardim2,vardim1,vardim0]

;***************************************************************
;  Make sure the dates are in an array
;***************************************************************

;  Number of different dates to pick up
numdates=n_elements(date)

;  Put the search dates in an array of dates called curdate
curdate=make_array(/string,numdates)
if numdates gt 1 then begin
  for i=0,numdates-1 do begin
    curdate[i]=strcompress(string(date[i]),/remove_all)
  endfor
endif else begin
  curdate[0]=strcompress(string(date),/remove_all)
endelse

;***************************************************************
;  Loop through the dates and make an array of file names
;***************************************************************

;  Loop through the dates
for i=0,numdates-1 do begin

  ;  Separate the date strings
  yy=strmid(curdate[i],0,4)
  mm=strmid(curdate[i],4,2)
  dd=strmid(curdate[i],6,2)

;  Arm file search string
  arm_file=arm_dir+datastream+'/'+yy+'/*.'+curdate[i]+'*.merged_sounding.cdf'

  ;  Unzip them if necessary
  unix_command='gunzip -fq '+arm_file+'*gz'
  spawn,unix_command

  ;  Get a list of the files for the day
  filesd=file_search(arm_file,count=num_files)  

  ;  Put these files in one big list if files exist for this date
  if num_files gt 0 then begin
    if n_elements(files) eq 0 then begin
      files=filesd
    endif else begin
      files=[files,filesd]
    endelse
  endif
endfor  ;end of loop through dates

;  Number of files to read and cat together
num_files=n_elements(files)

;*****************************************************
;  Loop through the files and read in the data
;*****************************************************

;  Loop through all the matching files in the directory
for f=0,num_files-1 do begin

;  Print the filename
  print,f,files[f]

;  Open the netcdf file
  cdf_id=ncdf_open(files[f])

;  Get the dimension ids
  time_did=ncdf_dimid(cdf_id,'time')
  height_did=ncdf_dimid(cdf_id,'height')

;  Get the dimension
  if time_did ne -1 then ncdf_diminq,cdf_id,time_did,char_strng,numtimes
  if height_did ne -1 then ncdf_diminq,cdf_id,height_did,char_strng,numheights

;  Create array for missing data.  Value selected at the top of the program
  if time_did ne -1 then dummy1d=make_array(numtimes,/float,value=missing)
  if height_did ne -1 and time_did ne -1 then $
    dummy2d=make_array(numheights,numtimes,/float,value=missing)

;  Get the variable ids
  for vv=0,n_elements(varname)-1 do begin
    xstr=varname[vv]+'_id=ncdf_varid(cdf_id,"'+varname[vv]+'")'
    result=execute(xstr)
    if dp eq 'yes' then print,varname[vv]
  endfor
  
;
;  Loop through the variable names.  Get the variable if it exists.
;  If it doesn't exist, set the variable equal to the dummy array.
;  

  for vv=0,n_elements(varname)-1 do begin

;  Determine the dimension of this variable
    xstr='idx=where(var eq "'+varname[vv]+'",count)'
    result=execute(xstr)
    if count eq 0 then begin 
      print,varname[vv],'variable not in list of acceptable variables'
      stop
    endif else begin
      dim=vardim[idx]
      if dim eq 1 then dumname='dummy1d' $
      else if dim eq 2 then dumname='dummy2d' 
    endelse
    if dp eq 'yes' then print,'processing ',varname[v],'dimension',dim

;  Get the variable
    if dim eq 1 or dim eq 2 then begin
      xstr='if '+varname[vv]+'_id gt -1 then ncdf_varget,cdf_id,'+$
        varname[vv]+'_id,'+varname[vv]+' else '+varname[vv]+'='+dumname
    endif else begin
      xstr='if '+varname[vv]+'_id gt -1 then ncdf_varget,cdf_id,'+$
        varname[vv]+'_id,'+varname[vv]    
    endelse
    result=execute(xstr)  

;  Transpose 2d variables
    if dim eq 2 then begin
      if dp eq 'yes' then print,'transposing',varname[vv]
      xstr=varname[vv]+'=transpose('+varname[vv]+')'
      result=execute(xstr)
    endif
  endfor

;
;  Combine the data 
;

  for vv=0,n_elements(varname)-1 do begin

    if varname[vv] eq 'time_offset' then begin
      if n_elements(com_btto) eq 0 then begin
        com_btto=long(base_time)+long(time_offset)
      endif else begin
        com_btto=[long(com_btto),long(base_time)+long(time_offset)]
      endelse
    endif

    if varname[vv] ne 'base_time' and varname[vv] ne 'time_offset' and $
       varname[vv] ne 'height' then begin
      xstr='if n_elements(com_'+varname[vv]+') eq 0 then com_'+$
            varname[vv]+'='+varname[vv]+' else com_'+varname[vv]+$
            '=[com_'+varname[vv]+','+varname[vv]+']'
      result=execute(xstr)
    endif

  endfor

;  Close the netcdf file
  ncdf_close,cdf_id
    
endfor  ;loop through number of matching file names

;******************************************
;  Remove the 'com_' prefix from the data array names
;******************************************

for vv=0,n_elements(varname)-1 do begin

  if varname[vv] eq 'base_time' then begin
    base_time=long(com_btto[0])  
  endif else if varname[vv] eq 'time_offset' then begin
    time_offset=long(com_btto)-long(base_time)        
  endif else if varname[vv] ne 'height' then begin
    xstr=varname[vv]+'=com_'+varname[vv]
    result=execute(xstr)
  endif

endfor  ;end of loop through variables

;********************************************
;  Zip up the data files
;********************************************

;unix_command='gzip -fq  '+arm_file  ;zip the data files
;spawn,unix_command

;*********************************************
;  Fill in the gaps in the data with missing data arrays
;*********************************************
if fill_flag eq 'yes' then begin

  ;  First check if this is necessary
  time_offset2=time_offset+time_offset[1]
  delta=time_offset2-time_offset
  result=where(delta ne 60,count)
  if count gt 0 then begin

  ;  Find the 2d variables
  for vv=0,n_elements(varname)-1 do begin
    xstr='idx=where(var eq "'+varname[vv]+'",count)'
    result=execute(xstr)
    if count eq 0 then begin 
      print,varname[vv],'variable not in list of acceptable variables'
      stop
    endif else begin
      dim=vardim[idx]
      ;if dim ge 2 then begin
      if dim eq 1 or dim eq 2 then begin
        if n_elements(var2d) eq 0 then begin
           var2d=varname[vv] 
        endif else begin
           var2d=[var2d,varname[vv]]
        endelse
      endif  ;end of this is a 2d var
    endelse  ;end of found this varname in the list
  endfor  ;end of loop through varnames

  ;  If there are 2d variables then continue to fill
  if n_elements(var2d) gt 0 then begin

  ;  Fill in the internal blanks
    for vv=0,n_elements(var2d)-1 do begin
      old_time_offset=time_offset
      xstr='insert_blanks,'+var2d[vv]+',old_time_offset,height,missing
      result=execute(xstr)
    endfor

    time_offset=old_time_offset

  ;  Fill in the blanks around the arrays
    curdate=date+'01'
    for vv=0,n_elements(var2d)-1 do begin
      old_time_offset=time_offset & old_base_time=base_time
      xstr='insert_blank_beginning,'+var2d[vv]+$
           ',height,old_time_offset,old_base_time,curdate,missing'
      result=execute(xstr)
    endfor

    time_offset=old_time_offset & base_time=old_base_time

  endif  ;fill the 2d variable

  endif  ;necessary to fill
endif  ;fill flag is yes

;*****************************************
;  Put the data in a structure to pass back out
;*****************************************

data={data,filename:'string'}
for vv=0,n_elements(varname)-1 do begin
  xstr='data=create_struct(data,"'+varname[vv]+'",'+varname[vv]+')'
  result=execute(xstr)
endfor  

return
end