pro get_smet_list,varname,data,site,date,missing,num_files

;***************************************************************
;  Inputs - set here or passed in through the command line
;***************************************************************

;  Site to process
;site='nsa'

;  Date to process
;date='20010207'
;date='20000417'

;  Variables to get
;varname=['base_time','time_offset','atmos_pressure',$
;         'vap_pres_mean','temp_mean','relh_mean']
;varname=['base_time','time_offset','T2M_AVG','RH2M_AVG',$
;           'VP2M_AVG','AtmPress','PcpRate']

;  Missing data flag 
;missing=-9999

;**************************************************************
;  Establish some constants
;**************************************************************

;  Save a copy of the original variables
varname_orig=varname

;  Print flag
dp='no'

;  Main arm site directory
arm_dir='/uufs/chpc.utah.edu/common/home/mace_grp/data/arm/'+site+'/'

;  Base_time and time_offset names are the same
base_timename='base_time'
time_offsetname='time_offset'

;  Arm data directory, datastream, and varnames depends on site
if site eq 'sgp' then begin
  limit_date=julday(4,1,2001,0,0,0)    
  datastream_1='sgp1smosE13.a0'
  datastream_2='sgp1smosE13.b1'

  dew_point_name='none'  
  pressure_name='bar_pres'        ;kPa
  vapor_pressure_name='vap_pres'  ;kPa
  temperature_name='temp'         ;C
  relative_humidity_name='rh'     ;%
  precip_name='precip'            ;mm
  wind_speed_name='wspd'          ;m/s
  wind_direction_name='wdir'      ;degree 
endif else if site eq 'nsa' then begin
  limit_date_12=julday(10,31,2003,0,0,0)
  limit_date_23=julday(09,18,2008,0,0,0)
  datastream_1='nsamettwrC1.a1'
  datastream_2='nsamettwr4hC1.b1'
  datastream_3='nsametC1.b1'

;  These will have to be picked up separately for each file
;  if check_date ge limit_date then begin
;    dew_point_name='DP2M_AVG'        ;C
;    pressure_name='AtmPress'         ;hPa
;    vapor_pressure_name='VP2M_AVG'   ;kPa  I convert to hPa
;    temperature_name='T2M_AVG'       ;C
;    relative_humidity_name='RH2M_AVG';%
;    precip_name='PcpRate'            ;mm/hr
;    wind_speed_name='WS2M_S_WVT'     ;m/s
;    wind_direction_name='WD2M_DU_WVT';degree
;  endif else begin
;    dew_point_name='dew_pt_temp_mean'     ;C
;    pressure_name='atmos_pressure'        ;hPa
;    vapor_pressure_name='vap_pres_mean'   ;hPa
;    temperature_name='temp_mean'          ;C
;    relative_humidity_name='relh_mean'    ;%
;    precip_name='none'
;    wind_speed_name='wind_spd_mean'       ;m/s
;    wind_direction_name='wind_dir_vec_avg';degree  
;  endelse
endif else if site eq 'twpc1' then begin
  ;  Units for surface_pressure
  ;  From the begining, the units are hPa.  
  ;  Starting on 20060708 01:00:00
  ;  the units are now kPa
  datastream='twpsmet60sC1.b1'

  dew_point_name='none'  
  pressure_name='atmos_pressure'           ;hPa;kPa starting 20060708 01:00
  vapor_pressure_name='vappress_mean'      ;hPa;kPa starting 20040803
  temperature_name='temp_mean'             ;C
  relative_humidity_name='relh_mean'       ;%
  precip_name='precip_mean'                ;mm/hr
  wind_speed_name='wind1_spd_arith_avg'    ;m/s
  wind_direction_name='wind1_dir_vec_avg';degree 
endif else if site eq 'twpc2' then begin
  datastream='twpsmet60sC2.b1'

  dew_point_name='none'  
  pressure_name='atmos_pressure'           ;hPa
  vapor_pressure_name='vappress_mean'      ;hPa
  temperature_name='temp_mean'             ;C
  relative_humidity_name='relh_mean'       ;%
  precip_name='precip_mean'                ;mm/hr
  wind_speed_name='wind1_spd_vec_avg'      ;m/s
  wind_direction_name='wind1_dir_vec_avg';degree 

endif else if site eq 'twpc3' then begin
  datastream='twpsmet60sC3.b1'	

  dew_point_name='none'  
  pressure_name='atmos_pressure'           ;kPa
  vapor_pressure_name='vappress_mean'      ;kPa
  temperature_name='temp_mean'             ;C
  relative_humidity_name='relh_mean'       ;%
  precip_name='precip_mean'                ;mm/hr
  wind_speed_name='lo_wind_spd_arith_avg'  ;m/s
  wind_direction_name='lo_wind_dir_vec_avg';degree 
endif else if site eq 'nim' then begin
  datastream='nimmetM1.b1'

  dew_point_name='none'  
  pressure_name='atmos_pressure'           ;hPa
  vapor_pressure_name='vappress_mean'      ;kPa
  temperature_name='temp_mean'             ;C
  relative_humidity_name='relh_mean'       ;%
  precip_name='precip_rate_mean'           ;mm/hr 
  wind_speed_name='wind_spd_vec_avg'       ;m/s 
  wind_direction_name='wind_dir_vec_avg'   ;degree 
endif

;***************************************************************
;  Make sure the dates are in an array
;***************************************************************

;  Number of different dates to pick up
numdates=n_elements(date)

;  Put the search dates in an array of dates called curdate
curdate=make_array(/string,numdates)
if numdates gt 1 then begin
  for i=0,numdates-1 do begin
    curdate[i]=strcompress(string(date[i]),/remove_all)
  endfor
endif else begin
  curdate[0]=strcompress(string(date),/remove_all)
endelse

;***************************************************************
;  Loop through the dates and make an array of file names
;***************************************************************

;  Loop through the dates
for i=0,numdates-1 do begin

  ;  Separate the date strings
  yy=strmid(curdate[i],0,4)
  mm=strmid(curdate[i],4,2)
  dd=strmid(curdate[i],6,2)

  ;  sgp and nsa had datastream name changes
  check_date=julday(mm,dd,yy,0,0,0)
  if site eq 'sgp' then begin
    if check_date ge limit_date then begin
      datastream=datastream_2
    endif else begin
      datastream=datastream_1
    endelse
  endif
  if site eq 'nsa' then begin
    if check_date ge limit_date_23 then begin
      datastream=datastream_3
    endif else if check_date ge limit_date_12 then begin
      datastream=datastream_2
    endif else begin
      datastream=datastream_1
    endelse
  endif

  ;  Create the filename
  arm_file=arm_dir+datastream+'/'+yy+'/'+datastream+'*.'+curdate[i]+'*cdf'

  ;  Unzip them if necessary
  unix_command='gunzip -fq '+arm_file+'*gz'
  spawn,unix_command

  ;  Get a list of the files for the day
  filesd=file_search(arm_file,count=num_files)  

  ;  Put these files in one big list if files exist for this date
  if num_files gt 0 then begin
    if n_elements(files) eq 0 then begin
      files=filesd
    endif else begin
      files=[files,filesd]
    endelse
  endif
endfor  ;end of loop through the dates


;  Number of files to read and cat together
num_files=n_elements(files)

;  Continue on if files were found
if num_files gt 0 then begin

;***************************************************************
;  Loop through all the matching files and read in the data
;***************************************************************

for i=0,num_files-1 do begin  

;  Print the filename
  if dp eq 'yes' then print,files[i]

;  Separate out the date parts
  parts=strsplit(files[i],datastream,/regex)
  year=fix(strmid(files[i],parts[2]+1,4))
  month=fix(strmid(files[i],parts[2]+5,2))
  day=fix(strmid(files[i],parts[2]+7,2))

;  Calculate check date to determine var names
  check_date=julday(month,day,year,0,0,0)

;  Choose the varname depending on date
  if site eq 'nsa' then begin
    if check_date ge limit_date_23 then begin
      dew_point_name='dew_point_mean'            ;C
      pressure_name='atmos_pressure'             ;kPa
      vapor_pressure_name='vapor_pressure_mean'  ;kPa
      temperature_name='temp_mean'               ;C
      relative_humidity_name='rh_mean'           ;%
      precip_name='pws_precip_rate_mean_1min'    ;mm/hr
      wind_speed_name='wspd_vec_mean'            ;m/s
      wind_direction_name='wdir_vec_mean'        ;degree
    endif else if check_date ge limit_date_12 then begin
      dew_point_name='DP2M_AVG'        ;C
      pressure_name='AtmPress'         ;hPa
      vapor_pressure_name='VP2M_AVG'   ;kPa
      temperature_name='T2M_AVG'       ;C
      relative_humidity_name='RH2M_AVG';%
      precip_name='PcpRate'            ;mm/hr
      wind_speed_name='WS2M_S_WVT'     ;m/s
      wind_direction_name='WD2M_DU_WVT';degree
    endif else if check_date lt limit_date_12 then begin
      dew_point_name='dew_pt_temp_mean'     ;C
      pressure_name='atmos_pressure'        ;hPa
      vapor_pressure_name='vap_pres_mean'   ;hPa
      temperature_name='temp_mean'          ;C
      relative_humidity_name='relh_mean'    ;%
      precip_name='none'
      wind_speed_name='wind_spd_mean'       ;m/s
      wind_direction_name='wind_dir_vec_avg';degree  
    endif
  endif

;  List of 1d variables in the file and their dimensions
;  These variables are (time)
  var1=[dew_point_name,$
        pressure_name,$
        vapor_pressure_name,$
        temperature_name,$
        relative_humidity_name,$
        precip_name,$
        wind_speed_name,$
        wind_direction_name]
  vardim1=make_array(n_elements(var1),/int,value=1)

;  Variables that aren't to be combined across files
  var0=[base_timename,time_offsetname]
  vardim0=make_array(n_elements(var0),/int,value=0)

;  Combine these all into one big array
  var=[var1,var0]
  vardim=[vardim1,vardim0]

;  Put the new varname in the varname array
  result=where(varname_orig eq 'DP2M_AVG' or $
               varname_orig eq 'dew_pt_temp_mean' or $
               varname_orig eq 'dew_point_mean',count)
  if count gt 0 then varname[result]=dew_point_name
  result=where(varname_orig eq 'AtmPress' or $
               varname_orig eq 'atmos_pressure',count)
  if count gt 0 then varname[result]=pressure_name
  result=where(varname_orig eq 'VP2M_AVG' or $
               varname_orig eq 'vap_pres_mean' or $
               varname_orig eq 'vapor_pressure_mean',count)
  if count gt 0 then varname[result]=vapor_pressure_name
  result=where(varname_orig eq 'T2M_AVG' or $
               varname_orig eq 'temp_mean',count)
  if count gt 0 then varname[result]=temperature_name
  result=where(varname_orig eq 'RH2M_AVG' or $
               varname_orig eq 'relh_mean' or $
               varname_orig eq 'rh_mean',count)
  if count gt 0 then varname[result]=relative_humidity_name
  result=where(varname_orig eq 'PcpRate' or $
               varname_orig eq 'none' or $
               varname_orig eq 'pws_precip_rate_mean_1min',count)
  if count gt 0 then varname[result]=precip_name
  result=where(varname_orig eq 'WS2M_S_WVT' or $
               varname_orig eq 'wind_spd_mean' or $
               varname_orig eq 'wspd_vec_mean',count)
  if count gt 0 then varname[result]=wind_speed_name
  result=where(varname_orig eq 'WD2M_DU_WVT' or $
               varname_orig eq 'wind_dir_vec_avg' or $
               varname_orig eq 'wdir_vec_mean',count)
  if count gt 0 then varname[result]=wind_direction_name

;  Open the netcdf file
  cdf_id=ncdf_open(files[i])

;  Get the dimension id's
  time_did=ncdf_dimid(cdf_id,'time')

;  Get the dimensions
  ncdf_diminq,cdf_id,time_did,charstring,numtimes

;  Create the dummy variables
  dummy1d=make_array(numtimes,/float,value=missing)

;  Get the variable ids
  for v=0,n_elements(varname)-1 do begin
    ;xstr=varname[v]+'_id=ncdf_varid(cdf_id,"'+varname[v]+'")'
    xstr="v"+varname[v]+"_id=ncdf_varid(cdf_id,'"+varname[v]+"')"
    result=execute(xstr)
    if dp eq 'yes' then print,varname[v]
  endfor
;
;  Loop through the variable names.  Get the variable if it exists.
;  If it doesn't exist, set the variable equal to the dummy array.
;
    for v=0,n_elements(varname)-1 do begin

;  Determine the dimension of this variable
      ;xstr='idx=where(var eq "'+varname[v]+'",count)'
      xstr="idx=where(var eq '"+varname[v]+"',count)"
      result=execute(xstr)
      if count eq 0 then begin 
        print,varname[v],'variable not in list of acceptable variables'
        stop
      endif else begin
        dim=vardim[idx]
        if dim eq 1 then dumname='dummy1d' $
        else if dim eq 2 then dumname='dummy2d' $
        else if dim eq 3 then dumname='dummy3d'
      endelse
      if dp eq 'yes' then print,'processing ',varname[v],' dimension',dim

;  Get the variable
      if dim ne 0 then begin
        xstr='if v'+varname[v]+'_id gt -1 then ncdf_varget,cdf_id,v'+$
          varname[v]+'_id,v'+varname_orig[v]+' else v'+varname_orig[v]+'='+dumname
      endif else begin
        xstr='if v'+varname[v]+'_id gt -1 then ncdf_varget,cdf_id,v'+$
          varname[v]+'_id,'+varname[v]   
      endelse
      result=execute(xstr)

;  Get the min-max values from the attributes
      if dim ne 0 then begin
        ;xstr="if v"+varname[v]+"_id gt -1 then ncdf_attget,cdf_id,v"+$
        ;  varname[v]+"_id,'minimum',"+varname_orig[v]+"_min"
        ;result=execute(xstr)
        ;xstr=varname_orig[v]+'_min=float(string('+varname_orig[v]+'_min))'
        ;result=execute(xstr)

        ;xstr="if v"+varname[v]+"_id gt -1 then ncdf_attget,cdf_id,v"+$
        ;  varname[v]+"_id,'maximum',"+varname_orig[v]+"_max"
        ;result=execute(xstr)
        ;xstr=varname_orig[v]+'_max=float(string('+varname_orig[v]+'_max))'
        ;result=execute(xstr)

        ;  Initialize to missing
        var_min=missing & var_max=missing

        ;  Try for the attribute 'minimum'
        xstr="if v"+varname[v]+"_id gt -1 then ncdf_attget,cdf_id,v"+$
          varname[v]+"_id,'minimum',var_min"
        result=execute(xstr)
        var_min=float(string(var_min))
        ;  If 'minimum' doesn't exist then try for 'valid_min'
        if n_elements(var_min) eq 0 or var_min eq missing then begin
          xstr="if v"+varname[v]+"_id gt -1 then ncdf_attget,cdf_id,v"+$
          varname[v]+"_id,'valid_min',var_min"
          result=execute(xstr)
        endif
        ;  If both don't exist, try for 'range_min'
        if n_elements(var_min) eq 0 or var_min eq missing then begin
          errname=strsplit(varname[v],'_mean',/extract,/regex)
          errname=errname[0]
          xstr=errname+"_out_of_range_error_id=ncdf_varid(cdf_id,'"+$
                errname+"_out_of_range_error')"
          result=execute(xstr)
          xstr="if "+errname+"_out_of_range_error_id gt -1 then ncdf_attget,cdf_id,"+$
          errname+"_out_of_range_error_id,'range_min',var_min"
          result=execute(xstr)
        endif
        
        ;  Make sure it is a float
        xstr=varname_orig[v]+'_min=float(string(var_min))'
        result=execute(xstr)
 
        ;  Try for the attribute 'maximum'
        xstr="if v"+varname[v]+"_id gt -1 then ncdf_attget,cdf_id,v"+$
          varname[v]+"_id,'maximum',var_max"
        result=execute(xstr)
        var_max=float(string(var_max))
        ;  If 'maximum' doesn't exist then try for 'valid_max'
        if n_elements(var_max) eq 0 or var_max eq missing then begin
          xstr="if v"+varname[v]+"_id gt -1 then ncdf_attget,cdf_id,v"+$
          varname[v]+"_id,'valid_max',var_max"
          result=execute(xstr)
        endif
        ;  If both don't exist, try for 'range_max'
        if n_elements(var_max) eq 0 or var_max eq missing then begin
          errname=strsplit(varname[v],'_mean',/extract,/regex)
          errname=errname[0]
          xstr=errname+"_out_of_range_error_id=ncdf_varid(cdf_id,'"+$
                errname+"_out_of_range_error')"
          result=execute(xstr)
          xstr="if "+errname+"_out_of_range_error_id gt -1 then ncdf_attget,cdf_id,"+$
          errname+"_out_of_range_error_id,'range_max',var_max"
          result=execute(xstr)
        endif

        ;  Make sure it is a float
        xstr=varname_orig[v]+'_max=float(string(var_max))'
        result=execute(xstr)  
        if dp eq 'yes' then print,float(string(var_min)),float(string(var_max)),errname

      endif
       
;  If NSA then select variables closest to the ground
;  [0,*]=40 m  [1,*]=20 m  [2,*]=10 m  [3,*]=2 m
;  wspd=reform(wspd[3,*])
      if site eq 'nsa' then begin
        xstr='dim_nsa=size(v'+varname_orig[v]+',/dimensions)'
        result=execute(xstr)
        dim_nsa=dim_nsa[0]
        if dim_nsa eq 4 and check_date lt limit_date_12 then begin
          xstr='v'+varname_orig[v]+'=reform(v'+varname_orig[v]+'[3,*])'
          result=execute(xstr)
        endif
      endif

;  If NSA then convert vapor pressure from kPa to hPa for the 
;   nsamettwr4hC1.b1 and nsametC1.b1 data stream
      if varname[v] eq 'VP2M_AVG' or $
         varname[v] eq 'vapor_pressure_mean' then begin
        xstr='idx=where('+varname_orig[v]+' ne missing,count)'
        result=execute(xstr)
        if count gt 0 then begin
          ;kPa converted to hPa
          xstr='v'+varname_orig[v]+'[idx]=v'+varname_orig[v]+'[result]*10.0'  
          result=execute(xstr)
        endif
      endif

;  Transpose 2d variables
      if dim gt 1 then begin
        if dp eq 'yes' then print,'transposing'
        xstr='v'+varname_orig[v]+'=transpose(v'+varname_orig[v]+')'
        result=execute(xstr)
      endif
    endfor

;
;  Combine the data 
;
    
    for v=0,n_elements(varname)-1 do begin

      if varname[v] eq 'time_offset' then begin
        if n_elements(com_btto) eq 0 then begin
          com_btto=long(base_time)+long(time_offset)
        endif else begin
          com_btto=[long(com_btto),long(base_time)+long(time_offset)]
        endelse
      endif
    
      if varname[v] ne 'base_time' and varname[v] ne 'time_offset' then begin
        xstr='if n_elements(com_'+varname_orig[v]+') eq 0 then com_'+$
              varname_orig[v]+'=v'+varname_orig[v]+' else com_'+varname_orig[v]+$
              '=[com_'+varname_orig[v]+',v'+varname_orig[v]+']'
        result=execute(xstr)
      endif

    endfor

;  Close the netcdf file
    ncdf_close,cdf_id
  endfor  ;loop through number of matching files

;***************************************************************
;  Remove the 'com_' prefix from the data array names
;***************************************************************

  for v=0,n_elements(varname)-1 do begin

    if varname[v] eq 'base_time' then begin
      base_time=long(com_btto[0])  
    endif else if varname[v] eq 'time_offset' then begin
      time_offset=long(com_btto)-long(base_time)        
    endif else begin
      xstr=varname_orig[v]+'=com_'+varname_orig[v]
      result=execute(xstr)
    endelse

  endfor  ;end of loop through variables

;******************************************
;  At sgp, 99999 is put in for missing surface pressure data.
;  Change this to our missing data flag
;
;  Fix some other problems with sgp data
;******************************************
 
  if site eq 'sgp' then begin
    ;  bad pressure value is 99999
    result=where(bar_pres eq 99999,count)
    if count gt 0 then bar_pres[result]=missing
    ;  bad temperature value is 6999
    result=where(temp eq 6999.0 or temp eq -6999.0,count)
    if count gt 0 then temp[result]=missing
    ;  bad rh value is 6999
    result=where(rh eq 6999.0 or rh eq -6999.0,count)
    if count gt 0 then rh[result]=missing
    ;  bad vapor pressure value is 6999
    result=where(vap_pres eq 6999.0 or vap_pres eq -6999.0,count)
    if count gt 0 then vap_pres[result]=missing

    ;  At sgp from 20060302-20060317 bar_pres is reported
    ;  in hpa instead of kpa.  This puts bar_pres a factor of
    ;  10 too high.
    result=where(bar_pres gt 500,count)
    if count gt 0 then begin
      bar_pres[result]=bar_pres[result]/10.0
      print,date,' sfc pres reported 10X too high, fixed it*******'
    endif

  endif

;*****************************************
;  Put the data in a structure to pass back out
;*****************************************

  data={data,filename:'string'}
  for v=0,n_elements(varname)-1 do begin
    if varname[v] eq 'time_offset' or varname[v] eq 'base_time' then begin
      xstr='data=create_struct(data,"'+varname[v]+'",'+varname[v]+')'
      result=execute(xstr)     
    endif else begin
      xstr='data=create_struct(data,"'+varname_orig[v]+'",'+varname_orig[v]+')'
      result=execute(xstr)  
      xstr='data=create_struct(data,"'+varname_orig[v]+'_min",'+varname_orig[v]+'_min)'
      result=execute(xstr)
      xstr='data=create_struct(data,"'+varname_orig[v]+'_max",'+varname_orig[v]+'_max)'
      result=execute(xstr)     
    endelse
  endfor  

  ;  Add in a variable to hold the files
  xstr='data=create_struct(data,"files",files)'
  result=execute(xstr)
  
;***************************************************************
;  Zip up the data files
;***************************************************************

  for i=0,num_files-1 do begin
    unix_command='gzip -fq  '+files[i]  ;zip the data files
    spawn,unix_command
  endfor

endif  ;process if files were found

return
end