;***************************************************************
;  Skyrad is at the twp and nsa sites
;***************************************************************

pro get_skyrad_list,varname,data,site,date,missing,num_files

;***************************************************************
;  Inputs - set here or passed in through the command line
;***************************************************************

;  Site to process
;site='twpc2'

;  Date to process
;date='20060302'

;  Variables to get
;varname=['base_time','time_offset','down_long_hemisp_shaded1',$
;          'down_short_hemisp','down_short_diffuse_hemisp','short_direct_normal']

;  Missing data flag 
;missing=-9999

;**************************************************************
;  Establish some constants
;**************************************************************

;  Save a copy of the original variables
varname_orig=varname

;  Print flag
dp='no'

;  Arm directory and filename depend on site
if site eq 'twpc1' then begin
  arm_dir='/data/mace/arm_data/twp/Manus_C1/'
  datastream='twpskyrad60sC1.b1'
endif else if site eq 'twpc2' then begin
  arm_dir='/data/mace/arm_data/twp/Nauru_C2/'
  datastream='twpskyrad60sC2.b1'
endif else if site eq 'twpc3' then begin
  arm_dir='/data/mace/arm_data/twp/Darwin_C3/'
  datastream='twpskyrad60sC3.b1'
endif else if site eq 'nsa' then begin
  arm_dir='/data/mace2/arm_data/nsa/'
  datastream='nsaskyrad60sC1.b1'
endif

;  Variable names
base_timename='base_time'
time_offsetname='time_offset'

;  These variable names changed on 20010401
;irt_name='irt1_mean';'sky_ir_temp'
;down_long_name='down_long_hemisp_shaded1';'pirs_mean'
;down_short_name='down_short_hemisp';'psp1_mean' 
;down_short_diffuse_name='down_short_diffuse_hemisp';'psps_mean'
;short_direct_normal_name='short_direct_normal';'nip_mean'

;  Date when the variable names changed
limit_date=julday(4,1,2001,0,0,0)

;***************************************************************
;  Make sure the dates are in an array
;***************************************************************

;  Number of different dates to pick up
numdates=n_elements(date)

;  Put the search dates in an array of dates called curdate
curdate=make_array(/string,numdates)
if numdates gt 1 then begin
  for i=0,numdates-1 do begin
    curdate[i]=strcompress(string(date[i]),/remove_all)
  endfor
endif else begin
  curdate[0]=strcompress(string(date),/remove_all)
endelse

;***************************************************************
;  Loop through the dates and make an array of file names
;***************************************************************

;  Loop through the dates
for i=0,numdates-1 do begin

  ;  Separate out the date parts
  year=strmid(curdate[i],0,4)

  ;  Create the filename
  arm_file=arm_dir+datastream+'/'+year+'/'+datastream+'*.'+curdate[i]+'*cdf'
  if dp eq 'yes' then print,arm_file

  ;  Unzip them if necessary
  unix_command='gunzip -fq '+arm_file+'*gz'
  spawn,unix_command

  ;  Get a list of the files for the day
  filesd=file_search(arm_file,count=num_files)  

  ;  Put these files in one big list if files exist for this date
  if num_files gt 0 then begin
    if n_elements(files) eq 0 then begin
      files=filesd
    endif else begin
      files=[files,filesd]
    endelse
  endif
endfor  ;end of loop through the dates

;  Number of files to read and cat together
num_files=n_elements(files)

;  Continue on if files were found
if num_files gt 0 then begin

;***************************************************************
;  Loop through all the matching files and read in the data
;***************************************************************

  for i=0,num_files-1 do begin  

    ;  Print the filename
    if dp eq 'yes' then print,files[i]

    ;  Separate out the date parts
    base_name=file_basename(files[i])
    parts=strsplit(base_name,'.',/extract)
    year=strmid(parts[2],0,4)
    month=strmid(parts[2],4,2)
    day=strmid(parts[2],6,2)

    ;  Calculate check date to determine var names
    check_date=julday(month,day,year,0,0,0)

    ;  Choose the varname depending on the date
    if check_date lt limit_date then begin
      irt_name='irt1_mean'
      down_long_name='pirs_mean'
      down_short_name='psp1_mean' 
      down_short_diffuse_name='psps_mean'
      short_direct_normal_name='nip_mean'
    endif else begin
      irt_name='sky_ir_temp'
      down_long_name='down_long_hemisp_shaded1'
      down_short_name='down_short_hemisp'
      down_short_diffuse_name='down_short_diffuse_hemisp'
      short_direct_normal_name='short_direct_normal'
    endelse

    ;  List of 1d variables in the file and their dimensions
    ;  These variables are (time)
    var1=[irt_name,$
          down_long_name,$
          down_short_name,$
          down_short_diffuse_name,$
          short_direct_normal_name]
    vardim1=make_array(n_elements(var1),/int,value=1)

    ;  Variables that aren't to be combined across files
    var0=[base_timename,time_offsetname]
    vardim0=make_array(n_elements(var0),/int,value=0)

    ;  Combine these all into one big array
    var=[var1,var0]
    vardim=[vardim1,vardim0]

    ;  Put the new varname in the varname array
    result=where(varname_orig eq 'irt1_mean' or $
                 varname_orig eq 'sky_ir_temp',count)
    if count gt 0 then varname[result]=irt_name
    result=where(varname_orig eq 'pirs_mean' or $
                 varname_orig eq 'down_long_hemisp_shaded',count)
    if count gt 0 then varname[result]=down_long_name
    result=where(varname_orig eq 'psp1_mean' or $
                 varname_orig eq 'down_short_hemisp',count)
    if count gt 0 then varname[result]=down_short_name
    result=where(varname_orig eq 'psps_mean' or $
                 varname_orig eq 'down_short_diffuse_hemisp',count)
    if count gt 0 then varname[result]=down_short_diffuse_name
    result=where(varname_orig eq 'short_direct_normal' or $
                 varname_orig eq 'nip_mean',count)
    if count gt 0 then varname[result]=short_direct_normal_name

    ;  Open the netcdf file
    cdf_id=ncdf_open(files[i])

    ;  Get the dimension id's
    time_did=ncdf_dimid(cdf_id,'time')

    ;  Get the dimensions
    ncdf_diminq,cdf_id,time_did,charstring,numtimes

    ;  Create the dummy variables
    dummy1d=make_array(numtimes,/float,value=missing)

    ;  Get the variable ids
    for v=0,n_elements(varname)-1 do begin
      ;xstr=varname[v]+'_id=ncdf_varid(cdf_id,"'+varname[v]+'")'
      xstr="v"+varname[v]+"_id=ncdf_varid(cdf_id,'"+varname[v]+"')"
      result=execute(xstr)
      if dp eq 'yes' then print,varname[v]
    endfor

    ;
    ;  Loop through the variable names.  Get the variable if it exists.
    ;  If it doesn't exist, set the variable equal to the dummy array.
    ;
    for v=0,n_elements(varname)-1 do begin

      ;  Determine the dimension of this variable
      ;xstr='idx=where(var eq "'+varname[v]+'",count)'
      xstr="idx=where(var eq '"+varname[v]+"',count)"
      result=execute(xstr)
      if count eq 0 then begin 
        print,varname[v],'variable not in list of acceptable variables'
        stop
      endif else begin
        dim=vardim[idx]
        if dim eq 1 then dumname='dummy1d' $
        else if dim eq 2 then dumname='dummy2d' $
        else if dim eq 3 then dumname='dummy3d'
      endelse
      if dp eq 'yes' then print,'processing ',varname[v],' dimension',dim

      ;  Get the variable
      if dim ne 0 then begin
        xstr='if v'+varname[v]+'_id gt -1 then ncdf_varget,cdf_id,v'+$
          varname[v]+'_id,v'+varname_orig[v]+' else v'+varname_orig[v]+$
          '='+dumname
      endif else begin
        xstr='if v'+varname[v]+'_id gt -1 then ncdf_varget,cdf_id,v'+$
          varname[v]+'_id,'+varname[v]   
      endelse
      result=execute(xstr)

      ;  Transpose 2d variables
      if dim gt 1 then begin
        if dp eq 'yes' then print,'transposing'
        xstr='v'+varname_orig[v]+'=transpose(v'+varname_orig[v]+')'
        result=execute(xstr)
      endif
    endfor  ;end of loop through variable names

;
;  Combine the data 
;

    for v=0,n_elements(varname)-1 do begin

      if varname[v] eq 'time_offset' then begin
        if n_elements(com_btto) eq 0 then begin
          com_btto=long(base_time)+long(time_offset)
        endif else begin
          com_btto=[long(com_btto),long(base_time)+long(time_offset)]
        endelse
      endif
    
      if varname[v] ne 'base_time' and varname[v] ne 'time_offset' then begin
        xstr='if n_elements(com_'+varname_orig[v]+') eq 0 then com_'+$
              varname_orig[v]+'=v'+varname_orig[v]+' else com_'+varname_orig[v]+$
              '=[com_'+varname[v]+',v'+varname_orig[v]+']'
        result=execute(xstr)
      endif

    endfor

;  Close the netcdf file
    ncdf_close,cdf_id

  endfor  ;loop through number of matching files

;***************************************************************
;  Remove the 'com_' prefix from the data array names
;***************************************************************

  for v=0,n_elements(varname)-1 do begin

    if varname[v] eq 'base_time' then begin
      base_time=long(com_btto[0])  
    endif else if varname[v] eq 'time_offset' then begin
      time_offset=long(com_btto)-long(base_time)        
    endif else begin
      xstr=varname_orig[v]+'=com_'+varname_orig[v]
      result=execute(xstr)
    endelse

  endfor  ;end of loop through variables

;*****************************************
;  Put the data in a structure to pass back out
;*****************************************

  data={data,filename:'string'}
  for v=0,n_elements(varname)-1 do begin
    if varname[v] eq 'time_offset' or varname[v] eq 'base_time' then begin
      xstr='data=create_struct(data,"'+varname[v]+'",'+varname[v]+')'
    endif else begin
      xstr='data=create_struct(data,"'+varname_orig[v]+'",'+varname_orig[v]+')'
    endelse

    result=execute(xstr)
  endfor  

  ;  Add in a variable to hold the files
  xstr='data=create_struct(data,"files",files)'
  result=execute(xstr)

;***************************************************************
;  Zip up the data files
;***************************************************************

  for i=0,num_files-1 do begin
    unix_command='gzip -fq  '+files[i]  ;zip the data files
    spawn,unix_command
  endfor

endif  ;process if files were found

return
end