#!/bin/csh
# Harvest ASCII files from SRL before converting to CDF

set thelog="/tmp/grabfiles_"$$".txt"

# LET files
cd /disks/stereodata/l2/ahead/let/

# Comment out these next four lines so that we keep the initial unprocessed files.
#
# if ( -e raw_wget_a.txt ) /bin/rm raw_wget_a.txt
# if ( -e processreport.txt ) /bin/rm processreport.txt
# if ( -e filelist.txt ) /bin/rm filelist.txt

# wget -nv -nH --cut-dirs=5 -L -m -erobots=off -R "index*" -A "*.txt" http://www.srl.caltech.edu/STEREO/DATA/Level1/Public/ahead/ >& raw_wget_a.txt

awk -f processfiles.awk < raw_wget_a.txt | csh >& processreport.txt
awk -f listfiles.awk < raw_wget_a.txt > filelist.txt
if ( ! -z filelist.txt ) mailx -s "New HETA files" marchant@ssl.berkeley.edu < filelist.txt