pro get_mwrp_list,varname,data,site,date

;***************************************************************
;  Inputs - set here or passed in through the command line
;***************************************************************

;  Site to process
;site='twpc3'

;  Date to process
;date='20060119'

;  Variables to get
;varname=['base_time','time_offset','lwp',$
;         'iwv']

;**************************************************************
;  Establish some constants
;**************************************************************

;  Missing data flag
missing=-8888

;  Print flag
dp='no'

;  List of 1d variables in the file and their dimensions
;  These variables are (time)
var1=[$
'time',$
'iwv',$
'lwp',$
'flag',$
'env_temperature',$
'env_pressure',$
'env_relhumidity']
vardim1=make_array(n_elements(var1),/int,value=1)

;  Variables that aren't to be combined across files
var0=['base_time','time_offset','latitude','longitude','altitude','elevation']
vardim0=make_array(n_elements(var0),/int,value=0)

;  Combine these all into one big array
var=[var1,var0]
vardim=[vardim1,vardim0]

;***************************************************************
;  Make sure the dates are in an array
;***************************************************************

;  Number of different dates to pick up
numdates=n_elements(date)

;  Put the search dates in an array of dates called curdate
curdate=make_array(/string,numdates)
if numdates gt 1 then begin
  for i=0,numdates-1 do begin
    curdate[i]=strcompress(strmid(string(date[i]),2),/remove_all)
  endfor
endif else begin
  curdate[0]=strcompress(strmid(string(date),2),/remove_all)
endelse

;***************************************************************
;  Set up strings to build the filename
;***************************************************************

;  Arm data directory, datastream, and varnames depends on site
if site eq 'sgp' then begin
  arm_dir='/data/mace3/arm_data/sgp/'
  datastream='sgp1mwravgC1.c1'
endif else if site eq 'nsa' then begin
  arm_dir='/data/mace2/arm_data/nsa/'
  datastream='nsamwrlosC1.a1'
endif else if site eq 'twpc1' then begin
  arm_dir='/data/mace/arm_data/twp/Manus_C1/'
  datastream='twpmwrlosC1.b1'
endif else if site eq 'twpc2' then begin
  arm_dir='/data/mace/arm_data/twp/Nauru_C2/'
  datastream='twpmwrlosC2.b1'
endif else if site eq 'twpc3' then begin
  arm_dir='/data/mace/arm_data/twp/Darwin_C3/'
  datastream='mwr_profiler'
  filetag='hatpro_dar_l1b'
endif else if site eq 'crystal' then begin
  arm_dir='/data/mace5/arm_data/crystal_face/western_ground_site/'
endif

;***************************************************************
;  Loop through the dates and get the filename search string
;***************************************************************

;  Get the directory and filename depending on site
for i=0,numdates-1 do begin

;  Arm file search string
  arm_file=arm_dir+datastream+'/'+curdate[i]+'*'+filetag+'*nc*'

;  Put these generic filename strings into an array
  if i eq 0 then begin
    list_files=arm_file
  endif else begin
    list_files=[list_files,arm_file]
  endelse
endfor

;***************************************************************
;  Loop through the list of generic filenames and get a list
;  of the real filenames.  Unzip these files if necessary
;***************************************************************

j=-1
for i=0,numdates-1 do begin

;  Unzip if necessary
  arm_file=list_files[i]
  unix_command='gunzip -fq '+arm_file
  spawn,unix_command

;  List of matching files
  files=file_search(arm_file,count=num_files)
  if num_files ge 1 then j=j+1

  if j eq 0 and num_files gt 0 then begin
    all_files=files
  endif else if j gt 0 and num_files gt 0 then begin
    all_files=[all_files,files]
  endif
endfor

;  Final array of all the files to read and cat together
files=all_files

;  Number of files to read and cat together
num_files=n_elements(files)

;***************************************************************
;  Loop through all the matching files and read in the data
;***************************************************************

for i=0,num_files-1 do begin  

;  Print the filename
  print,files[i]

;  Open the netcdf file
  cdf_id=ncdf_open(files[i])

;  Get the dimension id's
  time_did=ncdf_dimid(cdf_id,'time')

;  Get the dimensions
  ncdf_diminq,cdf_id,time_did,charstring,numtimes

;  Create the dummy variables
  dummy1d=make_array(numtimes,/float,value=missing)

;  Get the variable ids
  for v=0,n_elements(varname)-1 do begin
    xstr=varname[v]+'_id=ncdf_varid(cdf_id,"'+varname[v]+'")'
    result=execute(xstr)
    if dp eq 'yes' then print,varname[v]
  endfor
;
;  Loop through the variable names.  Get the variable if it exists.
;  If it doesn't exist, set the variable equal to the dummy array.
;
    for v=0,n_elements(varname)-1 do begin

;  Determine the dimension of this variable
      xstr='idx=where(var eq "'+varname[v]+'",count)'
      result=execute(xstr)
      if count eq 0 then begin 
        print,varname[v],'variable not in list of acceptable variables'
        stop
      endif else begin
        dim=vardim[idx]
        if dim eq 1 then dumname='dummy1d' $
        else if dim eq 2 then dumname='dummy2d' $
        else if dim eq 3 then dumname='dummy3d'
      endelse
      if dp eq 'yes' then print,'processing ',varname[v],' dimension',dim

;  Get the variable
      if dim ne 0 then begin
        xstr='if '+varname[v]+'_id gt -1 then ncdf_varget,cdf_id,'+$
          varname[v]+'_id,'+varname[v]+' else '+varname[v]+'='+dumname
      endif else begin
        xstr='if '+varname[v]+'_id gt -1 then ncdf_varget,cdf_id,'+$
          varname[v]+'_id,'+varname[v]   
      endelse
      result=execute(xstr)

;  Transpose 2d variables
      if dim gt 1 then begin
        if dp eq 'yes' then print,'transposing'
        xstr=varname[v]+'=transpose('+varname[v]+')'
        result=execute(xstr)
      endif
    endfor

;
;  Combine the data 
;

    for v=0,n_elements(varname)-1 do begin

      if varname[v] eq 'time_offset' then begin
        if n_elements(com_btto) eq 0 then begin
          com_btto=long(base_time)+long(time_offset)
        endif else begin
          com_btto=[long(com_btto),long(base_time)+long(time_offset)]
        endelse
      endif
    
      if varname[v] ne 'base_time' and varname[v] ne 'time_offset' then begin
        xstr='if n_elements(com_'+varname[v]+') eq 0 then com_'+$
              varname[v]+'='+varname[v]+' else com_'+varname[v]+$
              '=[com_'+varname[v]+','+varname[v]+']'
        result=execute(xstr)
      endif

    endfor

;  Close the netcdf file
    ncdf_close,cdf_id

  endfor  ;loop through number of matching files

;***************************************************************
;  Remove the 'com_' prefix from the data array names
;***************************************************************

for v=0,n_elements(varname)-1 do begin

  if varname[v] eq 'base_time' then begin
    base_time=long(com_btto[0])  
  endif else if varname[v] eq 'time_offset' then begin
    time_offset=long(com_btto)-long(base_time)        
  endif else begin
    xstr=varname[v]+'=com_'+varname[v]
    result=execute(xstr)
  endelse

endfor  ;end of loop through variables

;*****************************************
;  Put the data in a structure to pass back out
;*****************************************

data={data,filename:'string'}
for v=0,n_elements(varname)-1 do begin
  xstr='data=create_struct(data,"'+varname[v]+'",'+varname[v]+')'
  result=execute(xstr)
endfor  

;  Add in a variable to hold the files
xstr='data=create_struct(data,"files",files)'
result=execute(xstr)

;***************************************************************
;  Zip up the data files
;***************************************************************

;for i=0,num_files-1 do begin
;  unix_command='gzip -fq  '+files[i]  ;zip the data files
;  spawn,unix_command
;endfor

return
end