NASA Logo, National Aeronautics and Space Administration
CDAWeb
+ FEEDBACK
CDAWeb banner

Alternative Data Access Methods


           A small Bash shell script below that reads the CDAWeb daily file
         listing and retrieves new files (either ones matching patterns in
         choosefile, or get all new files except ones matching patterns in
         skipfiles).  [Remove the "-p" from xargs to automate.  wget can be
         replaced with "curl -O" (-O has opposite meanings with curl/wget)]

           ftp_cdaweb.sh:
           #!/bin/sh
           # HTTP new files from CDAWeb; 2016 June 2 Robert.M.Candey@nasa.gov
           # have to run twice the first time
           # outputs new_files and deleted_files from the difference of new
         and old CDAWeb file lists
           # also retrieves new files, skipping any patterns in skipfiles or
         selecting only
           #   files matching patterns in choosefiles
           # remove -p from xargs command to automate
           if [ -e cdaweb.diff ]; then
                    rm cdaweb.diff
           fi
           if [ -e deleted_files ]; then
                    rm deleted_files
           fi
           if [ -e new_files ]; then
                    rm new_files
           fi
           if [ -e cdaweb.curr ]; then
               mv cdaweb.curr cdaweb.prev
           fi
           wget -O - "http://cdaweb.gsfc.nasa.gov/filelist.gz" | gunzip | grep "\.cdf" | sort -k4>   cdaweb.curr
           #curl "http://cdaweb.gsfc.nasa.gov/filelist.gz" | gunzip | grep "\.cdf" | sort -k4>   cdaweb.curr
           diff cdaweb.prev cdaweb.curr>   cdaweb.diff
           egrep "^>   " cdaweb.diff>   deleted_files
           egrep "^<   " cdaweb.diff>   new_files
           if [ -e skipfiles ]; then
                if [ -e choosefiles ]; then
                    cat new_files|grep -v -f skipfiles |grep -f choosefiles |colrm 1 37|xargs -I{} -p -t wget "http://cdaweb.gsfc.nasa.gov/{}"
                else
               cat new_files|grep -v -f skipfiles |colrm 1 37|xargs -I{} -p -t wget "http://cdaweb.gsfc.nasa.gov/{}"
                fi
           elif [ -e choosefiles ]; then
               cat new_files|grep -f choosefiles |colrm 1 37|xargs -I{} -p -t wget "http://cdaweb.gsfc.nasa.gov/{}"
           else
               cat new_files|colrm 1 37|xargs -I{} -p -t wget "http://cdaweb.gsfc.nasa.gov/{}"
           fi

NASA Logo -
	    nasa.gov