pro get_mwr_list,varname,data,site,date,missing

;***************************************************************
;  Inputs - set here or passed in through the command line
;***************************************************************

;  Site to process
;site='sgp'

;  Date to process
;date='20080102'

;  Variables to get
;varname=['base_time','time_offset','tbsky23','liq','sky_ir_temp']

;  Missing data flag 
;missing=-8888

;**************************************************************
;  Establish some constants
;**************************************************************

;  Save a copy of the original variables
varname_orig=varname

;  Print flag
dp='no'

;  Main arm site directory
arm_dir='/uufs/chpc.utah.edu/common/home/mace_grp/data/arm/'+site+'/'

;  Base_time and time_offset names are the same
base_timename='base_time'
time_offsetname='time_offset'

;  varname changes
limit_date_1=julday(12,31,1999,0,0,0)
limit_date_2=julday(4,1,2001,0,0,0)

;  Arm data directory, datastream, and varnames depends on site
if site eq 'sgp' then begin
  datastream='sgpmwrlosC1.b1'  ;'sgp1mwravgC1.c1'

  tbsky23name='tbsky23'
  tbsky31name='tbsky31'
  vapname='vap'
  liqname='liq'
  ir_tempname='sky_ir_temp'
  wet_windowname='wet_window'
  precip_flagname='precip_flag'
  qc_tbsky23name='qc_tbsky23'
endif else if site eq 'nsa' then begin
  datastream='nsamwrlosC1.a1'

endif else if site eq 'twpc1' then begin
  datastream='twpmwrlosC1.b1'

  ;ir_temp until 4-1-2001 then sky_ir_temp
  tbsky23name='tbsky23'
  tbsky31name='tbsky31'
  vapname='vap'
  liqname='liq'
  ;ir_tempname=''
  wet_windowname='wet_window'
  precip_flagname='precip_flag'
  qc_tbsky23name='none'
endif else if site eq 'twpc2' then begin
  datastream='twpmwrlosC2.b1'

  tbsky23name='tbsky23'
  tbsky31name='tbsky31'
  vapname='vap'
  liqname='liq'
  ir_tempname='sky_ir_temp'
  wet_windowname='wet_window'
  precip_flagname='precip_flag'  
  qc_tbsky23name='qc_tbsky23'
endif else if site eq 'twpc3' then begin
  datastream='twpmwrlosC3.b1'

  tbsky23name='tbsky23'
  tbsky31name='tbsky31'
  vapname='vap'
  liqname='liq'
  ir_tempname='sky_ir_temp'
  wet_windowname='wet_window'
  precip_flagname='precip_flag'  
  qc_tbsky23name='qc_tbsky23'
endif 

;***************************************************************
;  Make sure the dates are in an array
;***************************************************************

;  Number of different dates to pick up
numdates=n_elements(date)

;  Put the search dates in an array of dates called curdate
curdate=make_array(/string,numdates)
if numdates gt 1 then begin
  for i=0,numdates-1 do begin
    curdate[i]=strcompress(string(date[i]),/remove_all)
  endfor
endif else begin
  curdate[0]=strcompress(string(date),/remove_all)
endelse

;**************************************************************
;  Make some changes in varname to allow for the different 
;  variable names
;**************************************************************

result=where(varname eq 'liq',count)
if count eq 1 then varname[result]=liqname

result=where(varname eq 'vap',count)
if count eq 1 then varname[result]=vapname

result=where(varname eq 'base_time',count)
if count eq 1 then varname[result]=base_timename

result=where(varname eq 'time_offset',count)
if count eq 1 then varname[result]=time_offsetname

result=where(varname eq 'precip_flag',count)
if count eq 1 then varname[result]=precip_flagname

result=where(varname eq 'qc_tbsky23',count)
if count eq 1 then varname[result]=qc_tbsky23name

;***************************************************************
;  Loop through the dates and make an array of file names
;***************************************************************

;  Loop through the dates
for i=0,numdates-1 do begin

  ;  Separate the date strings
  yy=strmid(curdate[i],0,4)
  mm=strmid(curdate[i],4,2)
  dd=strmid(curdate[i],6,2)

  ;  Create the filename
  arm_file=arm_dir+datastream+'/'+yy+'/'+datastream+'*.'+curdate[i]+'*cd*'

  ;  Unzip them if necessary
  unix_command='gunzip -fq '+arm_file+'*gz'
  spawn,unix_command

  ;  Get a list of the files for the day
  filesd=file_search(arm_file,count=num_files)

  ;  Put these files in one big list if files exist for this date
  if num_files gt 0 then begin
    if n_elements(files) eq 0 then begin
      files=filesd
    endif else begin
      files=[files,filesd]
    endelse
  endif
endfor


;  Number of files to read and cat together
num_files=n_elements(files)

;  Continue on if files were found
if num_files gt 0 then begin

;***************************************************************
;  Loop through all the matching files and read in the data
;***************************************************************

for i=0,num_files-1 do begin  

;  Print the filename
  if dp eq 'yes' then print,files[i]

;  Separate out the date parts
  parts=strsplit(files[i],datastream,/regex)
  year=fix(strmid(files[i],parts[2]+1,4))
  month=fix(strmid(files[i],parts[2]+5,2))
  day=fix(strmid(files[i],parts[2]+7,2))

;  Calculate check date to determine var names
  check_date=julday(month,day,year,0,0,0)

;  Choose the varname depending on date
  ;if site eq 'sgp' and check_date lt limit_date_1 then begin
  ;  tbsky23name='23tbsky'
  ;  tbsky31name='31tbsky'
  ;endif else if site eq 'sgp' and check_date ge limit_date_1 then begin
  ;  tbsky23name='tbsky23'
  ;  tbsky31name='tbsky31'
  ;endif 
  if site eq 'twpc1' and check_date lt limit_date_2 then begin
    ir_tempname='ir_temp'
  endif else if site eq 'twpc1' and check_date ge limit_date_2 then begin
    ir_tempname='sky_ir_temp'
  endif

;  List of 1d variables in the file and their dimensions
;  These variables are (time)
  var1=[tbsky23name,$
        tbsky31name,$
        vapname,$
        liqname,$
        ir_tempname,$
        wet_windowname,$
        precip_flagname,$
        qc_tbsky23name]
  vardim1=make_array(n_elements(var1),/int,value=1)

;  Variables that aren't to be combined across files
  var0=[base_timename,time_offsetname]
  vardim0=make_array(n_elements(var0),/int,value=0)

;  Combine these all into one big array
  var=[var1,var0]
  vardim=[vardim1,vardim0]

;  Put the new varname in the varname array
  result=where(varname_orig eq 'tbsky23' or varname_orig eq '23tbsky',count)
  if count gt 0 then varname[result]=tbsky23name
  result=where(varname_orig eq 'tbsky31' or varname_orig eq '31tbsky',count)
  if count gt 0 then varname[result]=tbsky31name
  result=where(varname_orig eq 'ir_temp' or $
               varname_orig eq 'sky_ir_temp',count)
  if count gt 0 then varname[result]=ir_tempname
  result=where(varname_orig eq 'wet_window' or $
               varname_orig eq 'water_flag_fraction',count)
  if count gt 0 then varname[result]=wet_windowname

;  Open the netcdf file
  cdf_id=ncdf_open(files[i])

;  Get the dimension id's
  time_did=ncdf_dimid(cdf_id,'time')

;  Get the dimensions
  ncdf_diminq,cdf_id,time_did,charstring,numtimes

;  Create the dummy variables
  dummy1d=make_array(numtimes,/float,value=missing)

;  Get the variable ids
  for v=0,n_elements(varname)-1 do begin
    ;xstr=varname[v]+'_id=ncdf_varid(cdf_id,"'+varname[v]+'")'
    xstr="v"+varname[v]+"_id=ncdf_varid(cdf_id,'"+varname[v]+"')"
    result=execute(xstr)
    if dp eq 'yes' then print,varname[v]
  endfor
;
;  Loop through the variable names.  Get the variable if it exists.
;  If it doesn't exist, set the variable equal to the dummy array.
;
    for v=0,n_elements(varname)-1 do begin

;  Determine the dimension of this variable
      ;xstr='idx=where(var eq "'+varname[v]+'",count)'
      xstr="idx=where(var eq '"+varname[v]+"',count)"
      result=execute(xstr)
      if count eq 0 then begin 
        print,varname[v],'variable not in list of acceptable variables'
        stop
      endif else begin
        dim=vardim[idx]
        if dim eq 1 then dumname='dummy1d' $
        else if dim eq 2 then dumname='dummy2d' $
        else if dim eq 3 then dumname='dummy3d'
      endelse
      if dp eq 'yes' then print,'processing ',varname[v],' dimension',dim

;  Get the variable
      if dim ne 0 then begin
        xstr='if v'+varname[v]+'_id gt -1 then ncdf_varget,cdf_id,v'+$
          varname[v]+'_id,v'+varname[v]+' else v'+varname[v]+'='+dumname
      endif else begin
        xstr='if v'+varname[v]+'_id gt -1 then ncdf_varget,cdf_id,v'+$
          varname[v]+'_id,'+varname[v]   
      endelse
      result=execute(xstr)

;  Transpose 2d variables
      if dim gt 1 then begin
        if dp eq 'yes' then print,'transposing'
        xstr=varname[v]+'=transpose('+varname[v]+')'
        result=execute(xstr)
      endif
    endfor

;
;  Combine the data 
;

    for v=0,n_elements(varname)-1 do begin

      if varname[v] eq 'time_offset' then begin
        if n_elements(com_btto) eq 0 then begin
          com_btto=long(base_time)+long(time_offset)
        endif else begin
          com_btto=[long(com_btto),long(base_time)+long(time_offset)]
        endelse
      endif
    
      if varname[v] ne 'base_time' and varname[v] ne 'time_offset' then begin
        xstr='if n_elements(com_'+varname[v]+') eq 0 then com_'+$
              varname[v]+'=v'+varname[v]+' else com_'+varname[v]+$
              '=[com_'+varname[v]+',v'+varname[v]+']'
        result=execute(xstr)
      endif

    endfor

;  Close the netcdf file
    ncdf_close,cdf_id

  endfor  ;loop through number of matching files

;***************************************************************
;  Remove the 'com_' prefix from the data array names
;***************************************************************

for v=0,n_elements(varname)-1 do begin

  if varname[v] eq 'base_time' then begin
    base_time=long(com_btto[0])  
  endif else if varname[v] eq 'time_offset' then begin
    time_offset=long(com_btto)-long(base_time)        
  endif else begin
    xstr='v'+varname[v]+'=com_'+varname[v]
    result=execute(xstr)
  endelse

endfor  ;end of loop through variables

;*****************************************
;  Put the data in a structure to pass back out
;*****************************************

data={data,filename:'string'}
for v=0,n_elements(varname)-1 do begin
  if varname[v] eq '23tbsky' then begin
    xstr='data=create_struct(data,"tbsky23",v'+varname[v]+')'
  endif else if varname[v] eq '31tbsky' then begin
    xstr='data=create_struct(data,"tbsky31",v'+varname[v]+')'
  endif else if varname[v] eq 'time_offset' or varname[v] eq 'base_time' then begin
    xstr='data=create_struct(data,"'+varname[v]+'",'+varname[v]+')'
  endif else begin
    xstr='data=create_struct(data,"'+varname[v]+'",v'+varname[v]+')'
  endelse

  result=execute(xstr)
endfor  

;  Add in a variable to hold the files
xstr='data=create_struct(data,"files",files)'
result=execute(xstr)

;***************************************************************
;  Zip up the data files
;***************************************************************

for i=0,num_files-1 do begin
  unix_command='gzip -fq  '+files[i]  ;zip the data files
  spawn,unix_command
endfor

endif  ;process if files were found

return
end