pro get_mpl_list,varname,data,site,date

;***************************************************************
;  Inputs - set here or passed in through the command line
;***************************************************************

;  Site to process
site='sgp'

;  Date to process
date='20000409'

;  Variables to get
varname=['base_time','time_offset','range_bins',$
         'preliminary_cbh','background_signal',$
         'height','detector_counts']

;**************************************************************
;  Establish some constants
;**************************************************************

;  Missing data flag
missing=-8888

;  Print flag
dp='no'

;  List of 2d variables in the file and their dimensions
;  This variable is (time,range_bins)
var2=[$
'detector_counts']
vardim2=make_array(n_elements(var2),/int,value=2)

;  List of 1d variables in the file and their dimensions
;  These variables are (time)
var1=[$
'shots_summed',$
'pulse_rep',$
'energy_monitor',$
'detector_temp',$
'filter_temp',$
'instrument_temp',$
'laser_temp',$
'voltage_05',$
'voltage_10',$
'voltage_15',$
'preliminary_cbh',$
'background_signal',$
'range_offset']
vardim1=make_array(n_elements(var1),/int,value=1)

;  Variables that aren't to be combined across files
var0=['base_time','time_offset','range_bins','height']
vardim0=make_array(n_elements(var0),/int,value=0)

;  Combine these all into one big array
var=[var2,var1,var0]
vardim=[vardim2,vardim1,vardim0]

;***************************************************************
;  Make sure the dates are in an array
;***************************************************************

;  Number of different dates to pick up
numdates=n_elements(date)

;  Put the search dates in an array of dates called curdate
curdate=make_array(/string,numdates)
if numdates gt 1 then begin
  for i=0,numdates-1 do begin
    curdate[i]=strcompress(string(date[i]),/remove_all)
  endfor
endif else begin
  curdate[0]=strcompress(string(date),/remove_all)
endelse

;***************************************************************
;  Set up strings to build the filename
;***************************************************************

;  Arm data directory and datastream depends on site
if site eq 'sgp' then begin
  arm_dir='/data/mace3/arm_data/sgp/'
  datastream='sgpmplC1.a1'
endif else if site eq 'nsa' then begin
  arm_dir='/data/mace2/arm_data/nsa/'
  datastream='nsamplC1.a1'
endif else if site eq 'twpc1' then begin
  arm_dir='/data/mace/arm_data/twp/Manus_C1/'
  datastream='twpmplC1.a1'
endif else if site eq 'twpc2' then begin
  arm_dir='/data/mace/arm_data/twp/Nauru_C2/'
  datastream='twpmplC2.a1'
endif else if site eq 'twpc3' then begin
  arm_dir='/data/mace/arm_data/twp/Darwin_C3/'
  datastream='twpmplC3.a1'
endif else if site eq 'crystal' then begin
  arm_dir='/data/mace5/arm_data/crystal_face/western_ground_site/'
endif

;***************************************************************
;  Loop through the dates and get the filename search string
;***************************************************************

;  Get the directory and filename depending on site
for i=0,numdates-1 do begin

;  Arm file search string
  arm_file=arm_dir+datastream+'/'+datastream+'*.'+curdate[i]+'*cd*'

;  Put these generic filename strings into an array
  if i eq 0 then begin
    list_files=arm_file
  endif else begin
    list_files=[list_files,arm_file]
  endelse
endfor

;***************************************************************
;  Loop through the list of generic filenames and get a list
;  of the real filenames.  Unzip these files if necessary
;***************************************************************

j=-1
for i=0,numdates-1 do begin

;  Unzip if necessary
  arm_file=list_files[i]
  unix_command='gunzip -fq '+arm_file
  spawn,unix_command

;  List of matching files
  files=file_search(arm_file,count=num_files)
  if num_files ge 1 then j=j+1

  if j eq 0 and num_files gt 0 then begin
    all_files=files
  endif else if j gt 0 and num_files gt 0 then begin
    all_files=[all_files,files]
  endif
endfor

;  Final array of all the files to read and cat together
files=all_files

;  Number of files to read and cat together
num_files=n_elements(files)

;***************************************************************
;  Loop through all the matching files and read in the data
;***************************************************************

for i=0,num_files-1 do begin

;  Print the filename
  print,files[i]

;  Open the netcdf file
  cdf_id=ncdf_open(files[i])

;  Get the dimension id's
  time_did=ncdf_dimid(cdf_id,'time')
  height_did=ncdf_dimid(cdf_id,'height')

;  Get the dimensions
  ncdf_diminq,cdf_id,time_did,charstring,ntimes
  ncdf_diminq,cdf_id,height_did,charstring,nheights

;  Create array for nodata value
  dummy1d=make_array(ntimes,/float,value=missing)
  dummy2d=make_array(ntimes,nheights,/float,value=missing)

;  Get the variable ids
  for v=0,n_elements(varname)-1 do begin
    xstr=varname[v]+'_id=ncdf_varid(cdf_id,"'+varname[v]+'")'
    result=execute(xstr)
    if dp eq 'yes' then print,varname[v]
  endfor

;
;  Loop through the variable names.  Get the variable if it exists.
;  If it doesn't exist, set the variable equal to the dummy array.
;

  for v=0,n_elements(varname)-1 do begin

;  Determine the dimension of this variable
    xstr='idx=where(var eq "'+varname[v]+'",count)'
    result=execute(xstr)
    if count eq 0 then begin 
      print,varname[v],'variable not in list of acceptable variables'
      stop
    endif else begin
      dim=vardim[idx]
      if dim eq 1 then dumname='dummy1d' $
      else if dim eq 2 then dumname='dummy2d' $
      else if dim eq 3 then dumname='dummy3d'
    endelse
    if dp eq 'yes' then print,'processing ',varname[v],'dimension',dim

;  Get the variable
    if dim ne 0 then begin
      xstr='if '+varname[v]+'_id gt -1 then ncdf_varget,cdf_id,'+$
        varname[v]+'_id,'+varname[v]+' else '+varname[v]+'='+dumname
    endif else begin
      xstr='if '+varname[v]+'_id gt -1 then ncdf_varget,cdf_id,'+$
        varname[v]+'_id,'+varname[v]    
    endelse
    result=execute(xstr)  

;  Transpose 2d variables
    if dim gt 1 then begin
      if dp eq 'yes' then print,'transposing'
      xstr=varname[v]+'=transpose('+varname[v]+')'
      result=execute(xstr)
    endif
  endfor

;
;  Combine the data 
;

  for v=0,n_elements(varname)-1 do begin

    if varname[v] eq 'time_offset' then begin
      if n_elements(com_btto) eq 0 then begin
        com_btto=long(base_time)+long(time_offset)
      endif else begin
        com_btto=[long(com_btto),long(base_time)+long(time_offset)]
      endelse
    endif
    
    if varname[v] ne 'base_time' and varname[v] ne 'time_offset' and $
       varname[v] ne 'Heights' then begin
      xstr='if n_elements(com_'+varname[v]+') eq 0 then com_'+$
            varname[v]+'='+varname[v]+' else com_'+varname[v]+$
            '=[com_'+varname[v]+','+varname[v]+']'
      result=execute(xstr)
    endif

  endfor
  
;  Close the netcdf file
  ncdf_close,cdf_id

endfor  ;end of looping though the matching files

;******************************************
;  Remove the 'com_' prefix from the data array names
;******************************************

for v=0,n_elements(varname)-1 do begin

  if varname[v] eq 'base_time' then begin
    base_time=long(com_btto[0])  
  endif else if varname[v] eq 'time_offset' then begin
    time_offset=long(com_btto)-long(base_time)        
  endif else if varname[v] ne 'Heights' then begin
    xstr=varname[v]+'=com_'+varname[v]
    result=execute(xstr)
  endif

endfor  ;end of loop through variables

;*****************************************
;  Put the data in a structure to pass back out
;*****************************************

data={data,filename:'string'}
for v=0,n_elements(varname)-1 do begin
  xstr='data=create_struct(data,"'+varname[v]+'",'+varname[v]+')'
  result=execute(xstr)
endfor  

;***************************************************************
;  Plot the data
;***************************************************************

do_plot='yes'
if do_plot eq 'yes' then begin

;
;  Find the size of the arrays
;
  numtimes=n_elements(time_offset)  ;number of times
  numheights=n_elements(height)     ;number of heights
;
;  Make array to hold the background and new signal information
;
  background=make_array(numtimes,2,/float,value=0)
  signal=make_array(numtimes,numheights,/float,value=0)
;
;  Loop though the shots and calculate background
;
  for t=0,numtimes-1 do begin

;  One shot
    shot=reform(detector_counts[t,*])

;  Calculate the average of the last 50 pts as background
    back_stat=moment(shot[numheights-50:numheights-2])
    background[t,0]=back_stat[0]  ;average
    background[t,1]=back_stat[1]  ;variance

;  Subtract the background from the shot
    signal[t,*]=detector_counts[t,*]-background[t,0]
  endfor  ;end of loop through shots
;
;  Replace negatives and zeros with really small number
;
  result=where(signal le 0.0,count)
  if count gt 0 then signal[result]=1.e-5
;
;  Perform an r squared correction
;
  for t=0,numtimes-1 do begin
  ;  signal[t,*]=(alog10(signal[t,*]))+(2.*alog10(height))
    signal[t,*]=(10.*alog10(signal[t,*]))+(20.*alog10(height))
  endfor
;
;  Data max and min
;
  if min(signal) le -20.0 then begin
    result=where(signal lt -20.0,count)
    if count gt 0 then signal[result]=-20.0
  endif
  if max(signal) gt 20.0 then begin
    result=where(signal gt 20.0,count)
    if count gt 0 then signal[result]=20.0
  endif
;
;  Calculate time array for plotting
;
;  Number of seconds in a day
  sec_per_day=24D*60D*60D

;  Get IDL julian day of day 1
  day1=julday(1,1,1970,0,0,0)
 
;  Get IDL julian day of the entire time array
  julian_day=double(day1 + (base_time+time_offset)/sec_per_day)

;  Check the dates
  caldat,julian_day,mm,dd,yy,hh,mi,ss
  print,yy[0],mm[0],dd[0],hh[0],mi[0],ss[0],'start time'
  print,yy[numtimes-1],mm[numtimes-1],dd[numtimes-1],hh[numtimes-1],$
    mi[numtimes-1],ss[numtimes-1],'end time'

;  Shorten the array
  result=where(height le 15,count)
  height=height[result]
  signal=signal[*,result]

;  Set up the plot
  window,0,xsize=600,ysize=600,/pixmap
  loadct,5
  ;gamma_ct,1.65
  !p.background=!d.n_colors-1
  dummy=label_date(date_format=['%H:%I'])

;  Postion plots
  xl=0.1 & xr=0.92
  yb=0.07 & yt=0.9
  sx=0.05 & sy=0.08
  numplots_x=1 & numplots_y=1
  position_plots,xl,xr,yb,yt,sx,sy,numplots_x,numplots_y,pos
  cbpos=pos
  cbpos[*,0]=cbpos[*,3]+0.07
  cbpos[*,2]=0.99

;  ***  Plot Signal
  dmax=max(signal)
  dmin=min(signal)
  dmax=20.0
  dmin=-20.0
  image=bytscl(signal,max=dmax,min=dmin)

  contour,signal,julian_day,height,/nodata,xstyle=4,ystyle=4,$
    position=pos[0,*]
  px=!x.window*!d.x_vsize
  py=!y.window*!d.y_vsize
  sx=px[1]-px[0]+1
  sy=py[1]-py[0]+1
  tv,congrid(image,sx,sy),px[0],py[0]
  contour,signal,julian_day,height,/noerase,/nodata,$
    xtickformat='label_date',xstyle=1,ystyle=1,$
    xtickunits='hour',xticklen=-0.02,yticklen=-0.02,$
    color=0,xtitle='signal',$
    ytitle='Height (km)',position=pos[0,*]

  oplot,julian_day,preliminary_cbh,color=255,$
    psym=4,symsize=0.5

  colorbar_fanning,maxrange=dmax,minrange=dmin,$
  title='signal',$
  ncolors=256,format='(F6.2)',$
  vertical=1,color=0,position=cbpos[0,*]

  image_name=datastream+'.gif'
  write_gif,image_name,tvrd()
  stop
endif  ;end of do_plot

return
end