pro get_ciret, curdate,site,datastream,input_flag

common ciret_data, base_time,time_offset,height,$
  z_mean,cld_base,top_base,iwc_mean,radius_effective,$
  opt_depth,cld_base_temp,emissivity,tb_diff,number_total,$
  clr_flx,bsfd,calc_sfd,flx_calc,reflect
;
;  Pieces to use to make the directory and filename
;
year=fix(strmid(curdate,0,4))
month=fix(strmid(curdate,4,2))
check_date=year+month/12.

if input_flag eq 'one' then begin
  if site eq 'twpc1' then begin
    ;no retrievels
  endif else if site eq 'twpc2' then begin
    armdatadir='/home/mace/sgp_dir/ci_ret1_twp2/mean_aeri_mean_dbze/'
    datastream='ci_retrieval'
  endif else if site eq 'nsa' then begin
    ;no retrievels yet
  endif else if site eq 'sgp' then begin
    armdatadir='/home/mace/sgp_dir/ci_ret1_fl/mean_aeri_mean_dbze/'
    datastream='ci_retrieval'
  endif
  armfile=armdatadir+'*'+curdate+'*'+datastream+'*cd*'
endif else if input_flag eq 'eos' then begin
  if site eq 'twpc1' then begin
    armdatadir='/data/mace/arm_data/twp/Manus_C1/'
    eosfilename='twpC1.eos_val_mace.c1'
    datastream='ci_retrieval'
  endif else if site eq 'twpc2' then begin
    armdatadir='/data/mace/arm_data/twp/Nauru_C2/'
    eosfilename='twpC2.eos_val_mace.c1'
    datastream='ci_retrieval'
  endif else if site eq 'nsa' then begin
    armdatadir='/data/mace2/arm_data/nsa/'
    eosfilename='nsaC1.eos_val_mace.c1'
    datastream='ci_retrieval'
  endif else if site eq 'sgp' then begin
    armdatadir='/data/mace3/arm_data/sgp/'
    eosfilename='sgpC1.eos_val_mace.c1'
    datastream='ci_retrieval'
  endif
  armfile=armdatadir+eosfilename+'/'+eosfilename+'.'+curdate+'*.cd*'
endif
;
;  Unzip the data files
;
unix_command='gunzip -fq '+armfile 
spawn,unix_command
;
;  Get a list of the matching filenames
;
print, 'filename ',armfile
files=findfile(armfile,count=count)
print, 'files ',files
print, 'count ',count
if count eq 0 then exit  ;no files for this day
;
;  Process files
;
j=-1  ;counts number of cdf files
for i=0,count-1 do begin  ;do for all the matching files in the directory
;
;  Don't try to process any gif files that may be in the directory
;
  if strpos(files[i],'gif') eq -1 and $
  strpos(files[i],'png') eq -1 then begin  
    j=j+1
;
;  Open the netcdf file
;
    cdfid=ncdf_open(files[i])
;
;  Get the dimension id's
;
    time_did=ncdf_dimid(cdfid,'time')
    height_did=ncdf_dimid(cdfid,'height')
;
;  Get the variable id's
;
    base_time_id=ncdf_varid(cdfid,'base_time')
    time_offset_id=ncdf_varid(cdfid,'time_offset')
    height_id=ncdf_varid(cdfid,'height')

    z_mean_id=ncdf_varid(cdfid,'Z_mean')
    cld_base_id=ncdf_varid(cdfid,'cld_base') 
    top_base_id=ncdf_varid(cdfid,'top_base') 
    iwc_mean_id=ncdf_varid(cdfid,'IWC_mean')
    radius_effective_id=ncdf_varid(cdfid,'radius_effective_mean')
    opt_depth_id=ncdf_varid(cdfid,'fl_vis_opt_depth_mean')
    cld_base_temp_id=ncdf_varid(cdfid,'cld_base_temp')
    emissivity_id=ncdf_varid(cdfid,'emissivity_mean')
    tb_diff_id=ncdf_varid(cdfid,'Tb_diff')
    number_total_id=ncdf_varid(cdfid,'Number_total_mean')
    clr_flx_id=ncdf_varid(cdfid,'est_clr_flx_obs')
    bsfd_id=ncdf_varid(cdfid,'bsfd')
    calc_sfd_id=ncdf_varid(cdfid,'calc_sfd')
    flx_calc_id=ncdf_varid(cdfid,'clr_flx_calc')
    reflect_id=ncdf_varid(cdfid,'ReflectivityBestEstimate')
    clutter_id=ncdf_varid(cdfid,'qc_ReflectivityClutterFlag')
    artifact_id=ncdf_varid(cdfid,'qc_RadarArtifacts')
;
;  Get the data
;
    if base_time_id ne -1 then ncdf_varget,cdfid,base_time_id,base_time
    if time_offset_id ne -1 then ncdf_varget,cdfid,time_offset_id,time_offset
    if height_id ne -1 then ncdf_varget,cdfid,height_id,height

    if z_mean_id ne -1 then ncdf_varget,cdfid,z_mean_id,z_mean
    if cld_base_id ne -1 then ncdf_varget,cdfid,cld_base_id,cld_base
    if top_base_id ne -1 then ncdf_varget,cdfid,top_base_id,top_base
    if iwc_mean_id ne -1 then ncdf_varget,cdfid,iwc_mean_id,iwc_mean
    if radius_effective_id ne -1 then $
       ncdf_varget,cdfid,radius_effective_id,radius_effective
    if opt_depth_id ne -1 then ncdf_varget,cdfid,opt_depth_id,opt_depth
    if cld_base_temp_id ne -1 then $
       ncdf_varget,cdfid,cld_base_temp_id,cld_base_temp
    if emissivity_id ne -1 then ncdf_varget,cdfid,emissivity_id,emissivity
    if tb_diff_id ne -1 then ncdf_varget,cdfid,tb_diff_id,tb_diff
    if number_total_id ne -1 then ncdf_varget,cdfid,number_total_id,number_total
    if clr_flx_id ne -1 then ncdf_varget,cdfid,clr_flx_id,clr_flx
    if bsfd_id ne -1 then ncdf_varget,cdfid,bsfd_id,bsfd
    if calc_sfd_id ne -1 then ncdf_varget,cdfid,calc_sfd_id,calc_sfd
    if flx_calc_id ne -1 then ncdf_varget,cdfid,flx_calc_id,flx_calc
    if reflect_id ne -1 then ncdf_varget,cdfid,reflect_id,reflect
    if clutter_id ne -1 then ncdf_varget, cdfid, clutter_id, clutter
    if artifact_id ne -1 then ncdf_varget,cdfid,artifact_id,artifact
;
;  Combine the data into big arrays
;
    if j eq 0 then begin  ;if it is the first file
      if time_offset_id ne -1 then com_btto=base_time+time_offset
      if z_mean_id ne -1 then com_z_mean=z_mean
      if cld_base_id ne -1 then com_cld_base=cld_base
      if top_base_id ne -1 then com_top_base=top_base
      if iwc_mean_id ne -1 then com_iwc_mean=iwc_mean
      if radius_effective_id ne -1 then com_radius_effective=radius_effective
      if opt_depth_id ne -1 then com_opt_depth=opt_depth
      if cld_base_temp_id ne -1 then com_cld_base_temp=cld_base_temp
      if emissivity_id ne -1 then com_emissivity=emissivity
      if tb_diff_id ne -1 then com_tb_diff=tb_diff
      if number_total_id ne -1 then com_number_total=number_total
      if clr_flx_id ne -1 then com_clr_flx=clr_flx
      if bsfd_id ne -1 then com_bsfd=bsfd
      if calc_sfd_id ne -1 then com_calc_sfd=calc_sfd
      if flx_calc_id ne -1 then com_flx_calc=flx_calc
      if reflect_id ne -1 then com_reflect=reflect
      if clutter_id ne -1 then com_clutter=clutter
      if artifact_id ne -1 then com_artifact=artifact
    endif else begin  ;if it is not the first file
      if time_offset_id ne -1 then com_btto=[com_btto,base_time+time_offset]
      if z_mean_id ne -1 then com_z_mean=[com_z_mean,z_mean]
      if cld_base_id ne -1 then com_cld_base=[com_cld_base,cld_base]
      if top_base_id ne -1 then com_top_base=[com_top_base,top_base]
      if iwc_mean_id ne -1 then com_iwc_mean=[com_iwc_mean,iwc_mean]
      if radius_effective_id ne -1 then $
         com_radius_effective=[com_radius_effective,radius_effective]
      if opt_depth_id ne -1 then com_opt_depth=[com_opt_depth,opt_depth]
      if cld_base_temp_id ne -1 then $
         com_cld_base_temp=[com_cld_base_temp,cld_base_temp]
      if emissivity_id ne -1 then com_emissivity=[com_emissivity,emissivity]
      if tb_diff_id ne -1 then com_tb_diff=[com_tb_diff,tb_diff]
      if number_total_id ne -1 then $
         com_number_total=[com_number_total,number_total]
      if clr_flx_id ne -1 then com_clr_flx=[com_clr_flx,clr_flx]
      if bsfd_id ne -1 then com_bsfd=[com_bsfd,bsfd]
      if calc_sfd_id ne -1 then com_calc_sfd=[com_calc_sfd,calc_sfd]
      if flx_calc_id ne -1 then com_flx_calc=[com_flx_calc,flx_calc]
      if reflect_id ne -1 then com_reflect=[com_reflect,reflect]
      if clutter_id ne -1 then com_clutter=[com_clutter,clutter]     
      if artifact_id ne -1 then com_artifact=[com_artifact,artifact]
    endelse  ;end of if it is the first file
;
;  Close netcdf file
;
    ncdf_close,cdfid
  endif  ;only process cdf files
endfor  ;loop through number of matching file names
;
;  Skip the rest if there are no variables
;
if j lt 0 then print, 'found no files'
if j lt 0 then exit
;
;  Skip the rest if there are no variables
;
if  z_mean_id eq -1 and cld_base_id eq -1 and iwc_mean_id eq -1 $
then begin
  if input_flag eq 'eos' then begin
    openw,15,'/data/mace4/arm_data/eos/eos_image_name.dat'
    printf,15,0
    close,15
  endif  
  print, 'found no variables'
;
;  Zip up the data files
; 
  unix_command='gzip -fq  '+armfile  ;zip the data files
  spawn,unix_command
  unix_command='gunzip -fq *gif.gz'  ;unzip any gif files unintentionally zipped
  spawn,unix_command
  unix_command='chmod 664 '+armfile  ;change permissions
  spawn,unix_command
  unix_command='chgrp met-mace '+armfile  ;change group
  spawn,unix_command
  exit
endif
;
;  Create the final arrays
;
if base_time_id ne -1 then base_time=com_btto[0]
if time_offset_id ne -1 then time_offset=com_btto-com_btto[0]
if z_mean_id ne -1 then z_mean=com_z_mean
if cld_base_id ne -1 then cld_base=com_cld_base
if top_base_id ne -1 then top_base=com_top_base
if iwc_mean_id ne -1 then iwc_mean=com_iwc_mean
if radius_effective_id ne -1 then radius_effective=com_radius_effective
if opt_depth_id ne -1 then opt_depth=com_opt_depth
if cld_base_temp_id ne -1 then cld_base_temp=com_cld_base_temp
if tb_diff_id ne -1 then tb_diff=com_tb_diff
if emissivity_id ne -1 then emissivity=com_emissivity
if number_total_id ne -1 then number_total=com_number_total
if clr_flx_id ne -1 then clr_flx=com_clr_flx
if bsfd_id ne -1 then bsfd=com_bsfd
if calc_sfd_id ne -1 then calc_sfd=com_calc_sfd
if flx_calc_id ne -1 then flx_calc=com_flx_calc
if reflect_id ne -1 then reflect=com_reflect
if clutter_id ne -1 then clutter=com_clutter
if artifact_id ne -1 then artifact=com_artifact
;
;  Use clutter and artifact flag to restrict sgp data
;
if clutter_id ne -1 and artifact_id ne -1 and site eq 'sgp' then begin
  result=where(clutter eq 2 and artifact eq 1,count)
  if count ne 0 then reflect[result]=-9999
  result=where(clutter eq 3 and artifact eq 0,count)
  if count ne 0 then reflect[result]=-9999
  ;result=where(clutter eq 7 and artifact eq 7,count);; now good
  ;if count ne 0 then reflect[result]=-9999
  result=where(clutter eq 3 and artifact eq 6 and reflect ne -8888,count)
  if count ne 0 then reflect[result]=-9999
  result=where(clutter eq 0 and artifact eq 0 and reflect ne -8888,count)
  if count ne 0 then reflect[result]=-9999
endif

;
;  Zip up the data files
; 
unix_command='gzip -fq  '+armfile  ;zip the data files
spawn,unix_command
unix_command='gunzip -fq *gif.gz'  ;unzip any gif files unintentionally zipped
spawn,unix_command
unix_command='chmod 664 '+armfile  ;change permissions
spawn,unix_command
unix_command='chgrp met-mace '+armfile  ;change group
spawn,unix_command

return
end