pro get_ecmwf_list,site,date_original,varname,data,missing

;***************************************************************
;  Inputs - set here or passed in through the command line
;***************************************************************

;  Site to process
;site='twpc1'

;  Date to process
;date='20030101'

;  Variables to get
;varname=['base_time','time_offset','domains',$
;         'lev','p','T','R']

;  Missing data flag
;missing=-9999

;**************************************************************
;  Establish some constants
;**************************************************************

;  Print flag
dp='no'

;  For the ecmwf you will always have to pick up 1 month
result=strlen(date_original)
if result eq 8 then begin
  date=strmid(date_original,0,6)
endif else begin
  date=date_origial
endelse

;  List of 3d variables in the file and their dimensions
;  This variable is (time,levels,domains)
var3=['p','u','v','T','q','l','i','a','R','w']
vardim3=make_array(n_elements(var3),/int,value=3)

;  List of 1d variables in the file and their dimensions
;  These variables are (time)
var1=[$
'idat',$
'itim',$
'vdat',$
'vtim']
vardim1=make_array(n_elements(var1),/int,value=1)

;  Variables that aren't to be combined across files
var0=['base_time','time_offset','domains','lev']
vardim0=make_array(n_elements(var0),/int,value=0)

;  Combine these all into one big array
var=[var3,var1,var0]
vardim=[vardim3,vardim1,vardim0]

;***************************************************************
;  Make sure the dates are in an array
;***************************************************************

;  Number of different dates to pick up
numdates=n_elements(date)

;  Put the search dates in an array of dates called curdate
curdate=make_array(/string,numdates)
if numdates gt 1 then begin
  for i=0,numdates-1 do begin
    curdate[i]=strcompress(string(date[i]),/remove_all)
  endfor
endif else begin
  curdate[0]=strcompress(string(date),/remove_all)
endelse

;***************************************************************
;  Set up strings to build the filename
;***************************************************************

;  Arm data directory and datastream depends on site
if site eq 'sgp' then begin
  arm_dir='/data/mace3/arm_data/sgp/'
  datastream='sgpecmwf/'
  arm_file='sgpecmwfvarX1.c1'
endif else if site eq 'nsa' then begin
  arm_dir='/data/mace2/arm_data/nsa/'
  datastream='nsaecmwf/'
  arm_file='nsaecmwfvarX1.c1'
endif else if site eq 'twpc1' then begin
  arm_dir='/data/mace/arm_data/twp/Manus_C1/'
  datastream='twpecmwfman/'
  arm_file='twpecmwfmanvarX1.c1'
  ;  The domain we want is 48, or the first element
endif else if site eq 'twpc2' then begin
  arm_dir='/data/mace/arm_data/twp/Nauru_C2/'
  datastream='twpecmwfnau/'
  arm_file='twpecmwfnauvarX1.c1'
endif else if site eq 'twpc3' then begin
  arm_dir='/data/mace/arm_data/twp/Darwin_C3/'
  datastream='twpecmwfdar/'
  arm_file='twpecmwfdarvarX1.c1'
endif

;***************************************************************
;  Loop through the dates and get the filename search string
;***************************************************************

;  Get the directory and filename depending on site
for i=0,numdates-1 do begin

;  Arm file search string
  arm_file=arm_dir+datastream+'/'+arm_file+'*'+curdate[i]+'*cd*'

;  Put these generic filename strings into an array
  if i eq 0 then begin
    list_files=arm_file
  endif else begin
    list_files=[list_files,arm_file]
  endelse
endfor

;***************************************************************
;  Loop through the list of generic filenames and get a list
;  of the real filenames.  Unzip these files if necessary
;***************************************************************

j=-1
for i=0,numdates-1 do begin

;  Unzip if necessary
  arm_file=list_files[i]
  unix_command='gunzip -fq '+arm_file
  spawn,unix_command

;  List of matching files
  files=file_search(arm_file,count=num_files)
  if num_files ge 1 then j=j+1

  if j eq 0 and num_files gt 0 then begin
    all_files=files
  endif else if j gt 0 and num_files gt 0 then begin
    all_files=[all_files,files]
  endif
endfor

;  Final array of all the files to read and cat together
files=all_files

;  Number of files to read and cat together
num_files=n_elements(files)

;***************************************************************
;  Loop through all the matching files and read in the data
;***************************************************************

for i=0,num_files-1 do begin

;  Print the filename
  print,files[i]

;  Open the netcdf file
  cdf_id=ncdf_open(files[i])

;  Get the dimension ids
  time_did=ncdf_dimid(cdf_id,'time')
  levels_did=ncdf_dimid(cdf_id,'levels')
  domains_did=ncdf_dimid(cdf_id,'domains')

;  Get the dimensions
  ncdf_diminq,cdf_id,time_did,charstring,numtimes
  ncdf_diminq,cdf_id,levels_did,charstring,numlevels
  ncdf_diminq,cdf_id,domains_did,charstring,numdomains

;  Create array for nodata value
  dummy1d=make_array(numtimes,/float,value=missing)
  dummy3d=make_array(numtimes,numlevels,numdomains,/float,value=missing)

;  Get the variable ids
  for j=0,n_elements(varname)-1 do begin
    xstr=varname[j]+'_id=ncdf_varid(cdf_id,"'+varname[j]+'")'
    result=execute(xstr)
    if dp eq 'yes' then print,varname[j]
  endfor

;
;  Loop through the variable names.  Get the variable if it exists.
;  If it doesn't exist, set the variable equal to the dummy array.
;

  for j=0,n_elements(varname)-1 do begin

;  Determine the dimension of this variable
    xstr='idx=where(var eq "'+varname[j]+'",count)'
    result=execute(xstr)
    if count eq 0 then begin 
      print,varname[j],'variable not in list of acceptable variables'
      stop
    endif else begin
      dim=vardim[idx]
      if dim eq 1 then dumname='dummy1d' $
      else if dim eq 2 then dumname='dummy2d' $
      else if dim eq 3 then dumname='dummy3d'
    endelse
    if dp eq 'yes' then print,'processing ',varname[v],'dimension',dim

;  Get the variable
    if dim ne 0 then begin
      xstr='if '+varname[j]+'_id gt -1 then ncdf_varget,cdf_id,'+$
        varname[j]+'_id,'+varname[j]+' else '+varname[j]+'='+dumname
    endif else begin
      xstr='if '+varname[j]+'_id gt -1 then ncdf_varget,cdf_id,'+$
        varname[j]+'_id,'+varname[j]    
    endelse
    result=execute(xstr)  

;  Select out the domain over the arm site 
    if dim eq 3 then begin

      xstr=varname[j]+'=transpose('+varname[j]+')'
      result=execute(xstr)

      xstr=varname[j]+'=reverse('+varname[j]+',2)'
      result=execute(xstr)

      xstr=varname[j]+'=reform('+varname[j]+'[*,*,0])'
      result=execute(xstr)

    endif
  endfor

;
;  Combine the data 
;

  for j=0,n_elements(varname)-1 do begin

    if varname[j] eq 'time_offset' then begin
      if n_elements(com_btto) eq 0 then begin
        com_btto=long(base_time)+long(time_offset)
      endif else begin
        com_btto=[long(com_btto),long(base_time)+long(time_offset)]
      endelse
    endif
    
    if varname[j] ne 'base_time' and varname[j] ne 'time_offset' and $
       varname[j] ne 'domains' and varname[j] ne 'lev' then begin
      xstr='if n_elements(com_'+varname[j]+') eq 0 then com_'+$
            varname[j]+'='+varname[j]+' else com_'+varname[j]+$
            '=[com_'+varname[j]+','+varname[j]+']'
      result=execute(xstr)
    endif

  endfor
  
;  Close the netcdf file
  ncdf_close,cdf_id

endfor  ;end of looping though the matching files

;******************************************
;  Remove the 'com_' prefix from the data array names
;******************************************

for j=0,n_elements(varname)-1 do begin

  if varname[j] eq 'base_time' then begin
    base_time=long(com_btto[0])  
  endif else if varname[j] eq 'time_offset' then begin
    time_offset=long(com_btto)-long(base_time)        
  endif else if varname[j] ne 'Heights' then begin
    xstr=varname[j]+'=com_'+varname[j]
    result=execute(xstr)
  endif

endfor  ;end of loop through variables

;*****************************************
;  Put the data in a structure to pass back out
;*****************************************

data={data,filename:'string'}
for j=0,n_elements(varname)-1 do begin
  xstr='data=create_struct(data,"'+varname[j]+'",'+varname[j]+')'
  result=execute(xstr)
endfor  

;*****************************************
;  Zip up the files
;*****************************************

for i=0,num_files-1 do begin

  unix_command='gzip '+files[i]
  spawn,unix_command

endfor


;
;  Calculate height from pressure
;
;if month eq '03' then pressure=press_1[fix(numtimes/2),*] else  pressure=press_1[0,*]
;To=288  ;Kelvins
;Po=1013.25  ;mb
;G=6.50  ;deg km-1
;G=G/1000.  ;deg m-1
;grav=9.81  ;m/s
;R=287  ;J/deg/kg
;exponent=R*G/grav
;bracket=1-(pressure/Po)^exponent
;height=To/G*bracket
;height=height/1000.  ;convert to km


return
end