;
;  Read in the mwrlos data 
;
pro get_mwrlos_list,varname,data,site,date,missing,num_files

;***************************************************************
;  Inputs - set here or passed in through the command line
;***************************************************************

;  Site to process
;site='sgp'

;  Date to process
;date='19970615'

;  Variables to get
;varname=['base_time','time_offset','tbsky23','liq']

;  Missing data flag 
;missing=-8888

;**************************************************************
;  Establish some constants
;**************************************************************

;  Print flag
dp='no'

;  Main arm site directory
arm_dir='/uufs/chpc.utah.edu/common/home/mace_data3/data/arm/'+site+'/'

;  Data directory and filename depend on the site
if site eq 'sgp' then begin
  datastream='sgpmwrlosC1.b1'
endif else if site eq 'nsa' then begin
  datastream='nsamwrlosC1.b1'
endif else if site eq 'twpc1' then begin
  datastream='twpmwrlosC1.b1'
endif else if site eq 'twpc2' then begin
  datastream='twpmwrlosC2.b1'
endif else if site eq 'twpc3' then begin
  datastream='twpmwrlosC3.b1'
endif 

;  List of 1d variables in the file and their dimensions
var1=['tbsky23',$
      'tbsky31',$
      'vap',$
      'liq',$
      'sky_ir_temp',$
      'wet_window',$
      'precip_flag',$
      'qc_tbsky23']
vardim1=make_array(n_elements(var1),/int,value=1)

;  Variables that aren't to be combined across files
var0=['base_time','time_offset']
vardim0=make_array(n_elements(var0),/int,value=0)

;  Combine these all into one big array
var=[var1,var0]
vardim=[vardim1,vardim0]

;***************************************************************
;  Make sure the dates are in an array
;***************************************************************

;  Number of different dates to pick up
numdates=n_elements(date)

;  Put the search dates in an array of dates called curdate
curdate=make_array(/string,numdates)
if numdates gt 1 then begin
  for i=0,numdates-1 do begin
    curdate[i]=strcompress(string(date[i]),/remove_all)
  endfor
endif else begin
  curdate[0]=strcompress(string(date),/remove_all)
endelse


;***************************************************************
;  Loop through the dates and make an array of file names
;***************************************************************

;  Loop through the dates
for i=0,numdates-1 do begin

  ;  Separate out the date parts
  year=strmid(curdate[i],0,4)

  ;  Create the filename
  arm_file=arm_dir+datastream+'/'+year+'/'+datastream+'*.'+curdate[i]+'*cdf*'
  if dp eq 'yes' then print,arm_file

  ;  Unzip them if necessary
  unix_command='gunzip -fq '+arm_file+'*gz'
  spawn,unix_command

  ;  Get a list of the files for the day
  filesd=file_search(arm_file,count=num_files)

  ;  Put these files in one big list if files exist for this date
  if num_files gt 0 then begin
    if n_elements(files) eq 0 then begin
      files=filesd
    endif else begin
      files=[files,filesd]
    endelse
  endif
endfor  ;end of loop through the dates

;  Number of files to read and cat together
num_files=n_elements(files)

;***************************************************************
;  Only continue processing if files were found
;***************************************************************

if num_files gt 0 then begin

;***************************************************************
;  Loop through all the matching files and read in the data
;***************************************************************

  ;  Loop through all the matching files in the directory
  for i=0,num_files-1 do begin  

    ;  Print the filename
    if dp eq 'yes' then print,files[i]

    ;  Open the netcdf file
    cdf_id=ncdf_open(files[i])

    ;  Get the dimension id's
    time_did=ncdf_dimid(cdf_id,'time')

    ;  Get the dimensions
    ncdf_diminq,cdf_id,time_did,charstring,numtimes

    ;  Create the dummy variables
    dummy1d=make_array(numtimes,/float,value=missing)

    ;  Get the variable ids
    for v=0,n_elements(varname)-1 do begin
      xstr=varname[v]+'_id=ncdf_varid(cdf_id,"'+varname[v]+'")'
      result=execute(xstr)
      if dp eq 'yes' then print,varname[v]
    endfor

    ;
    ;  Loop through the variable names.  Get the variable if it exists.
    ;  If it doesn't exist, set the variable equal to the dummy array.
    ;

    for v=0,n_elements(varname)-1 do begin

      ;  Determine the dimension of this variable
      xstr='idx=where(var eq "'+varname[v]+'",count)'
      result=execute(xstr)
      if count eq 0 then begin 
        print,varname[v],'variable not in list of acceptable variables'
        stop
      endif else begin
        dim=vardim[idx]
        if dim eq 1 then dumname='dummy1d' $
        else if dim eq 2 then dumname='dummy2d' $
        else if dim eq 3 then dumname='dummy3d'
      endelse
      if dp eq 'yes' then print,'processing ',varname[v],' dimension',dim

      ;  Get the variable
      if dim ne 0 then begin
        xstr='if '+varname[v]+'_id gt -1 then ncdf_varget,cdf_id,'+$
          varname[v]+'_id,'+varname[v]+' else '+varname[v]+'='+dumname
      endif else begin
        xstr='if '+varname[v]+'_id gt -1 then ncdf_varget,cdf_id,'+$
          varname[v]+'_id,'+varname[v]   
      endelse
      result=execute(xstr)

      ;  Transpose 2d variables
      if dim gt 1 then begin
        if dp eq 'yes' then print,'transposing'
        xstr=varname[v]+'=transpose('+varname[v]+')'
        result=execute(xstr)
      endif
    endfor

    ;
    ;  Combine the data 
    ;

    for v=0,n_elements(varname)-1 do begin

      if varname[v] eq 'time_offset' then begin
        if n_elements(com_btto) eq 0 then begin
          com_btto=long(base_time)+long(time_offset)
        endif else begin
          com_btto=[long(com_btto),long(base_time)+long(time_offset)]
        endelse
      endif
    
      if varname[v] ne 'base_time' and varname[v] ne 'time_offset' then begin
        xstr='if n_elements(com_'+varname[v]+') eq 0 then com_'+$
              varname[v]+'='+varname[v]+' else com_'+varname[v]+$
              '=[com_'+varname[v]+','+varname[v]+']'
        result=execute(xstr)
      endif

    endfor

    ;  Close the netcdf file
    ncdf_close,cdf_id

  endfor  ;loop through number of matching files

;***************************************************************
;  Remove the 'com_' prefix from the data array names
;***************************************************************

  for v=0,n_elements(varname)-1 do begin

    if varname[v] eq 'base_time' then begin
      base_time=long(com_btto[0])  
    endif else if varname[v] eq 'time_offset' then begin
      time_offset=long(com_btto)-long(base_time)        
    endif else begin
      xstr=varname[v]+'=com_'+varname[v]
      result=execute(xstr)
    endelse

  endfor  ;end of loop through variables

;*****************************************
;  Put the data in a structure to pass back out
;*****************************************

  data={data,filename:'string'}
  for v=0,n_elements(varname)-1 do begin
    xstr='data=create_struct(data,"'+varname[v]+'",'+varname[v]+')'
    result=execute(xstr)
  endfor  

  ;  Add in a variable to hold the files
  xstr='data=create_struct(data,"files",files)'
  result=execute(xstr)

;***************************************************************
;  Zip up the data files
;***************************************************************

  for i=0,num_files-1 do begin
    unix_command='gzip -fq  '+files[i]  ;zip the data files
    spawn,unix_command
  endfor

endif  ;end of found files so continue processing

return
end