pro get_clouds2, curdate,site,datastream,input_flag

common clouds_data, base_time, time_offset, height,$
	clutter,cbase,ctop,reflect,velocity,spectrum,artifact
;
;  Pieces to use to make the directory and filename
;
if input_flag eq 'one' then begin
  if site eq 'twpc1' then begin
	armdatadir='/data/mace/arm_data/twp/Manus_C1/'
	datastream='twpmmcrcalC1.a1'
        armfile='twpmmcrMergedMomentsC1.a1'
  endif else if site eq 'twpc2' then begin
	armdatadir='/data/mace/arm_data/twp/Nauru_C2/'
	datastream='twpmmcrcalC2.a1'
        armfile='twpmmcrMergedMomentsC2.a1'
  endif else if site eq 'nsa' then begin
	armdatadir='/data/mace2/arm_data/nsa/'
	datastream='nsammcrcalC1.a1'
        armfile='nsammcrMergedMomentsC1.a1'
  endif else if site eq 'sgp' then begin
	armdatadir='/data/mace3/arm_data/sgp/'
	datastream='sgpmmcrcalC1.a1'
        armfile='sgpmmcrMergedMomentsC1.a1'
  endif
  armfile=armdatadir+datastream+'/'+armfile+'.'+curdate+'*.cd*'
endif else if input_flag eq 'eos' then begin
  if site eq 'twpc1' then begin
	armdatadir='/data/mace/arm_data/twp/Manus_C1/'
	eosfilename='twpC1.eos_val_mace.c1'
	datastream='twpmmcrMergedMomentsC1.a1'
  endif else if site eq 'twpc2' then begin
	armdatadir='/data/mace/arm_data/twp/Nauru_C2/'
	eosfilename='twpC2.eos_val_mace.c1'
	datastream='twpmmcrMergedMomentsC2.a1'
  endif else if site eq 'nsa' then begin
	armdatadir='/data/mace2/arm_data/nsa/'
	eosfilename='nsaC1.eos_val_mace.c1'
	datastream='nsammcrMergedMomentsC1.a1'
  endif else if site eq 'sgp' then begin
	armdatadir='/data/mace3/arm_data/sgp/'
	eosfilename='sgpC1.eos_val_mace.c1'
	datastream='sgpmmcrMergedMomentsC1.a1'
  endif
  armfile=armdatadir+eosfilename+'/'+eosfilename+'.'+curdate+'*.cd*'
endif
;
;  Unzip the data files
;
unix_command='gunzip -fq '+armfile 
spawn,unix_command
;
;  Get a list of the matching filenames
;
print, 'filename ',armfile
files=findfile(armfile,count=count)
print, 'files ',files
print, 'count ',count
if count eq 0 then exit  ;no files for this day
;
;  Process files
;
j=-1  ;counts number of cdf files
for i=0,count-1 do begin  ;do for all the matching files in the directory
;
;  Don't try to process any gif or png files that may be in the directory
;
  if strpos(files[i],'gif') eq -1 and $
  strpos(files[i],'png') eq -1 then begin  
    j=j+1
;
;  Open the netcdf file
;
    cdfid=ncdf_open(files[i])
;
;  Get the dimension id's
;
    time_did=ncdf_dimid(cdfid,'time')
    if input_flag eq 'eos' then height_did=ncdf_dimid(cdfid,'height') else $
    height_did=ncdf_dimid(cdfid,'nheights')
;
;  Get the variable id's
;
    base_time_id=ncdf_varid(cdfid,'base_time')
    time_offset_id=ncdf_varid(cdfid,'time_offset')
    if input_flag eq 'eos' then height_id=ncdf_varid(cdfid,'height') else $
    height_id=ncdf_varid(cdfid,'Heights')
    reflect_id=ncdf_varid(cdfid,'ReflectivityBestEstimate')
    velocity_id=ncdf_varid(cdfid,'MeanDopplerVelocity')
    spectrum_id=ncdf_varid(cdfid,'SpectralWidth')
    cbase_id=ncdf_varid(cdfid,'CloudBaseBestEstimate')
    ctop_id=ncdf_varid(cdfid,'RadarFirstTop')
    clutter_id=ncdf_varid(cdfid,'qc_ReflectivityClutterFlag')
    artifact_id=ncdf_varid(cdfid,'qc_RadarArtifacts')
;
;  Get the data
;
    if base_time_id ne -1 then ncdf_varget, cdfid, base_time_id, base_time 
    if time_offset_id ne -1 then ncdf_varget, cdfid, time_offset_id, time_offset
    if height_id ne -1 then ncdf_varget, cdfid, height_id, height  
    if reflect_id ne -1 then ncdf_varget, cdfid, reflect_id, reflect  
    if velocity_id ne -1 then ncdf_varget, cdfid, velocity_id, velocity 
    if spectrum_id ne -1 then ncdf_varget, cdfid, spectrum_id, spectrum  
    if cbase_id ne -1 then ncdf_varget, cdfid, cbase_id, cbase  
    if ctop_id ne -1 then ncdf_varget, cdfid, ctop_id, ctop 
    if clutter_id ne -1 then ncdf_varget, cdfid, clutter_id, clutter
    if artifact_id ne -1 then ncdf_varget,cdfid,artifact_id,artifact
;
;  Combine the data into big arrays
;
    if j eq 0 then begin  ;if it is the first file
      if time_offset_id ne -1 then com_btto=base_time+time_offset
      if reflect_id ne -1 then com_reflect=reflect
      if velocity_id ne -1 then com_velocity=velocity
      if spectrum_id ne -1 then com_spectrum=spectrum
      if cbase_id ne -1 then com_cbase=cbase
      if ctop_id ne -1 then com_ctop=ctop
      if clutter_id ne -1 then com_clutter=clutter
      if artifact_id ne -1 then com_artifact=artifact
    endif else begin  ;if it is not the first file
      if time_offset_id ne -1 then com_btto=[com_btto,base_time+time_offset]
      if reflect_id ne -1 then com_reflect=[com_reflect,reflect]
      if velocity_id ne -1 then com_velocity=[com_velocity,velocity]
      if spectrum_id ne -1 then com_spectrum=[com_spectrum,spectrum]
      if cbase_id ne -1 then com_cbase=[com_cbase,cbase]
      if ctop_id ne -1 then com_ctop=[com_ctop,ctop]
      if clutter_id ne -1 then com_clutter=[com_clutter,clutter]     
      if artifact_id ne -1 then com_artifact=[com_artifact,artifact]
    endelse  ;end of if it is the first file
;
;  Close the netcdf file
;
    ncdf_close,cdfid
  endif  ;only process cdf files
endfor  ;loop through number of matching file names
;
;  Skip the rest if there are no files
;
if j lt 0 then print, 'found no files'
if j lt 0 then exit
;
;  Skip the rest if there are no variables
;
if reflect_id eq -1 and velocity_id eq -1 and spectrum_id eq -1 $
and cbase_id eq -1 and ctop_id eq -1 and clutter_id eq -1 $
then begin
  if input_flag eq 'eos' then begin
    openw,15,'/data/mace4/arm_data/eos/eos_image_name.dat'
    printf,15,0
    close,15
  endif  
  print, 'found no variables'
;
;  Zip up the data files
; 
  unix_command='gzip -fq  '+armfile  ;zip the data files
  spawn,unix_command
  unix_command='gunzip -fq *gif.gz'  ;unzip any gif files unintentionally zipped
  spawn,unix_command
  unix_command='chmod 664 '+armfile  ;change permissions
  spawn,unix_command
  unix_command='chgrp met-mace '+armfile  ;change group
  spawn,unix_command
  exit  ;end of found no variables
endif
;
;  Create the final arrays
;
if base_time_id ne -1 then base_time=com_btto[0]
if time_offset_id ne -1 then time_offset=com_btto-com_btto[0]
if reflect_id ne -1 then reflect=com_reflect
if velocity_id ne -1 then velocity=com_velocity
if spectrum_id ne -1 then spectrum=com_spectrum
if cbase_id ne -1 then cbase=com_cbase
if ctop_id ne -1 then ctop=com_ctop
if clutter_id ne -1 then clutter=com_clutter
if artifact_id ne -1 then artifact=com_artifact
;
;  Use clutter flag to restrict sgp data
;
;if clutter_id ne -1 and site eq 'sgp' then begin
;  result=where(clutter eq 0 or clutter eq 3 or clutter eq 7 $
;  or clutter eq 2,count)
;  if count ne 0 then reflect[result]=-9999
;endif
;
;  Use clutter and artifact flag to restrict sgp data
;
if clutter_id ne -1 and artifact_id ne -1 and site eq 'sgp' then begin
  result=where(clutter eq 2 and artifact eq 1,count)
  if count ne 0 then reflect[result]=-9999
  result=where(clutter eq 3 and artifact eq 0,count)
  if count ne 0 then reflect[result]=-9999
  ;result=where(clutter eq 7 and artifact eq 7,count);; now good
  ;if count ne 0 then reflect[result]=-9999
  result=where(clutter eq 3 and artifact eq 6 and reflect ne -8888,count)
  if count ne 0 then reflect[result]=-9999
  result=where(clutter eq 0 and artifact eq 0 and reflect ne -8888,count)
  if count ne 0 then reflect[result]=-9999
endif
;
;  Zip up the data files
; 
unix_command='gzip -fq  '+armfile  ;zip the data files
spawn,unix_command
unix_command='gunzip -fq *gif.gz'  ;unzip any gif files unintentionally zipped
spawn,unix_command
unix_command='chmod 664 '+armfile  ;change permissions
spawn,unix_command
unix_command='chgrp met-mace '+armfile  ;change group
spawn,unix_command

return
end