#! /bin/sh # This is a shell archive, meaning: # 1. Remove everything above the #! /bin/sh line. # 2. Save the resulting text in a file. # 3. Execute the file with /bin/sh (not csh) to create the files: # ftptools.l # behead # ftplist # ftpfind # ftpget # grabfiles # ship # unpackmail # This archive created: Mon Aug 15 13:45:15 1994 export PATH; PATH=/bin:$PATH if test -f 'ftptools.l' then echo shar: will not over-write existing file "'ftptools.l'" else cat << \SHAR_EOF > 'ftptools.l' .\" $Id$ .TH FTPTOOLS 1 "13 Aug 1994" "" .SH NAME ftptools \- Scripts to aid in performing unattended \fIftp\fP transfers: ftplist ftpfind ftpget grabfiles ship unpackmail behead .SH SYNOPSIS ftplist [-h \fIremotehost\fP ] [-d \fIremotedirectory\fP] [-r] [-v] ftpfind ftpget [-h \fIremotehost\fP ] [-d \fIremotedirectory\fP]... [-f \fIremotefile:localfile\fP]... [-m \fIfilepattern\fP] [-c \fIlocaldirectory\fP] [-v] grabfiles [-v] ship [-btoa | -rscs | -benc | -uu] \fIfilename recipient\fP unpackmail behead [\fIfiles\fP] .TP 8 NOTE When using these tools, please be courteous to sites that you're affecting; things like ftpfind should preferably be run late at night or early in the morning, or on weekends. Be careful with things you ship around; email is not really meant to distribute large files reliably. Shipping copies of very large programs by email is one way of guaranteeing one's unpopularity if things break (spool areas on intermediate machines fill up, mailers on the way dumping core, doing multiple resends, etc.) Consider using snail mail to send tapes, using uucp to call the remote system, or some similar means in preference to mail. .SH DESCRIPTION .LP Additional descriptions are available in the comment-headers of the scripts themselves. .TP 8 ftplist [-h \fIremotehost\fP ] [-d \fIremotedirectory\fP] [-r] [-v] Lists the contents of \fIremotedirectory\fP on host \fIremotehost\fP, recursively (using \fIls -lR\fP), if -r is specified. .TP 8 ftpfind .\"Leave the line above & below, blank or the formatting will screw-up Reformats output of \fIftplist\fP that has been run on a remote file-system to look like that of \fIfind\(1\)\fP. .TP 8 ftpget [-h \fIremotehost\fP ] [-d \fIremotedirectory\fP]... [-f \fIremotefile:localfile\fP]... [-m \fIfilepattern\fP] [-c \fIlocaldirectory\fP] [-v] Performs batch anonymous ftp. Essentially converts a list of of command-line arguments into input to ftp. .TP 8 grabfiles [-v] Takes the output of \fIftpgrep\fP and gets those files by generating anonymous ftp scripts and running them. This allows one to \fIftpgrep\fP for something, extract the lines that one wants and feed them to this script. (The extraction may be done with \fIgrep(1), sed(1), awk(1),\fP or \fIcut(1)\fP and \fIpaste(1)\fP.) .TP 8 ship [-btoa | -rscs | -benc | -uu] \fIfilename recipient\fP Send \fIfilename\fP, a binary file, via email to \fIrecipient\fP. The appropriate encoding (e.g. \fIbtoa\fP, \fIrscsencode\fP, or \fIuuencode\fP) is applied, and the file is split into managable parts and each part is wrapped for easy reconstruction.Compressed files, tar files \ or compressed tar files may be sent in this manner. If \fIfilename\fP is a \ pathname, then all leading components will be removed from the \fIfilename\fP \ before it is sent. .TP 8 unpackmail \- Takes mail messages in from a mailbox, assumes they are the pieces of a file that has been split and mailed with \fIship\fP, and unpacks it. \fIWARNING: DO NOT RUN THIS COMMAND ON MAIL YOU DO NOT TRUST. IT USES COMMANDS FOUND IN THE MAIL MESSAGE, AND IS THEREFORE INSECURE.\fP .TP 8 behead [\fIfiles\fP] Strips off the header from a mail or News file i.e. everything up to the first empty line. This simplifies unsharing packages. .SH OPTIONS .TP 8 -h \fIremotehost\fP Specifies \fIremotehost\fP as the name of the remote host. The default is the local host-name. .TP 8 -d \fIremotedirectory\fP Specifies the remote directory to list using \fIdir\fP. Multiple \fI\-d\fP options will list each \fIremotedirectory\f in turn after \fIcd\fP'ing into each of them. The argument to \fI\-d\fP is passed to the \fIcd\fP command that is executed on the remote host. If the paths are relative, ensure that the sequence is correct, also beware of \fIcd\fP'ing into directories that are symbolic-links\( a common occurence nowadays \), and \fIcd\fP'ing back out. .TP 8 -r Specifies that the directories are to be listed recursively on the remote-host, \ using \fIls \-lR\fP, instead of \fIdir\fP which lists the current directory only. .TP 8 -v Turns on the verbose option of \fIftp\fP, and shows all responses from the ftp server. .TP 8 -f \fIremotefile[:localfile]\fP Gets \fIremotefile\fP and places it in \fIlocalfile\fP. .TP 8 -m \fIpattern\fP Performs an \fImget\fP (see \fIftp(1)\fP) with \fIpattern\fP as the argument. Remember to quote \fIpattern\fP to prevent shell-expansion. .TP 8 -c \fIlocaldirectory\fP Performs a \fIcd\fP to \fIlocaldirectory\fP. (Equivalent to \fIlcd\fP). .SH EXAMPLES Using \fIftplist\fP: .nf \fI ftplist -h ftp.x.org -d /contrib -d /R5contrib \fP .fi will list directories \fI/contrib\fP and \fI/R5contrib\fP at the archive site \fIftp.x.org\fP. Using \fIftpget\fP: .nf \fI ftpget -h ftp.x.org -d /R5contrib -f aXe-6.0.tar.Z:aXe-6.0.tar.Z -d /pub/DOCS -c ~/DOCS -m '*'\fP .fi will get \fIaXe-6.0.tar.Z\fP from \fI/R5contrib\fP on \fIftp.x.org\fP, and put it in \fIaXe-6.0.tar.Z\fP in the current working directory on the local system, and get all the files from \fI/pub/DOCS\fP and put them in the local \fI~/DOCS\fP directory. .LP The sequence of the options supplied is very important, since the equivalent commands are executed on the remote-host by \fIftp(1)\fP in the exact order specified. .LP It is more convenient to create a shell-script called, perhaps, ftpscript, that encapsulates the \fIftpget\IP command and that is run by \fIcron(1)\fP: .nf #!/bin/sh # ftpscript: a cron shell-script wrapper for unattended ftp'ing with # ftpget. # Author: elf@ee.ryerson.ca # #the hostname; uncomment the one that is to be called HNAME=ftp.x.org #HNAME=wuarchive.wustl.edu #HNAME=prep.ai.mit.edu #HNAME=gatekeeper.dec.com #HNAME=shape.mps.ohio-state.edu #the file to get; only if using "get" rather than "mget" FNAME=lemacs-19.3-sun4.tar.Z #the location of the file on the remote host; uncomment the one that #is relevant #GDIR=/msdos/Games/Apogee #GDIR=pub/gnu/lucid GDIR=/contrib #GDIR=pub/gnu #the target dir on our system TDIR=/home/eccles/elf/tmp/ftpstuff cd $TDIR #single-file get version #/usr/local/bin/ftpget -h $HNAME -d $GDIR -f $FNAME:$FNAME -v #mget version /usr/local/bin/ftpget -h $HNAME -d $GDIR -m 'mpeg-*' -v .fi .SH SEE ALSO ftp(1), cron(1), crontab(1), find(1), grep(1), less(1), tar(1), \ awk(1), btoa(1) compress(1), uuencode(1), shar(l), unshar(l), archie(l) .SH AUTHORS Mark Moraes , with the following exceptions \fIship\fP: Jean-Francois Lamy & Mark Moraes \fIftptools.l\fP (this man-page): Luis Fernandes .SH COPYRIGHT This software is provided as-is, with no warranties whatsoever. You're free to do whatever you like with it - neither the author nor the University of Toronto are responsible for the consequences of use of this software, no matter how awful, even if these consequences are the result of flaws in the software. If you find bugs, or make improvements, I'd like to know. Please document any modifications you make clearly, identifying yourself. \- Mark Moraes \- Luis Fernandes SHAR_EOF fi # end of overwriting check if test -f 'behead' then echo shar: will not over-write existing file "'behead'" else cat << \SHAR_EOF > 'behead' #! /bin/sh # Strips off the header from a mail/News message i.e. till the first # empty line # Mark Moraes, University of Toronto if [ $# -eq 0 ]; then sed -e '1,/^$/d' -e '/^[ ]*$/d' else for i do sed -e '1,/^$/d' -e '/^[ ]*$/d' $i done fi SHAR_EOF chmod +x 'behead' fi # end of overwriting check if test -f 'ftplist' then echo shar: will not over-write existing file "'ftplist'" else cat << \SHAR_EOF > 'ftplist' #! /bin/sh # $Id: ftplist,v 1.3 91/05/07 21:15:50 moraes Exp $ # Script to list a remote anonymous ftp directory's contents # -h specifies the remote host (default localhost) # -d specifies the remote directory to list with the dir command - you can # provide a sequence of -d options - they will be listed in turn, after # cd'ing to each of them. (the argument to -d is the argument to the # cd command executed at the other end) # If the paths are relative, make sure you get the sequence right. # Be careful with relative paths - there are far too many symlinks nowadays. # (default is the ftp login directory) # -r switches on recursive mode. ls -lR is used instead of dir for listing. # -v turns on the verbose option of ftp, and shows all responses from the # ftp server. # For example, # ftplist -h expo.lcs.mit.edu -d contrib -d ../pub/R3/fixes # will list directories ~ftp/contrib and ~ftp/pub/R3/fixes on expo.lcs.mit.edu # Mark Moraes , Feb 1, 1989 PATH=/local/bin:/usr/ucb:/usr/bin:/bin export PATH SITE=`hostname` # Just to get a fully qualified hostname. If you don't have the host # program, you should get it! The next line can be deleted with no # ill-effects. SITE=`host $SITE | awk 'NR == 1 {print $1; exit}'` usage="Usage: $0 [-h remotehost] [-d remotedirectory]... [-v] [-r]" writeflag= ftpflags="-i -n" verbflag= ls="dir" set x `getopt vrh:d: $*` if [ $? != 0 ]; then echo $usage exit 1 fi shift for i in $* do case $i in -v) verbflag=-v; shift;; -r) ls="ls -lR"; shift;; -h) remhost=$2; shift 2;; -d) remdir="$remdir $2"; shift 2;; --) shift; break;; esac done if [ x${verbflag} != x ]; then ftpflags="${ftpflags} -v" fi if [ x${remhost} = x ]; then remhost=localhost fi remdir=${remdir-'.'} if [ $# -ne 0 ]; then echo ${usage} exit 1 fi set - $remdir (echo user anonymous ${USER-ftp}@${SITE} for i do echo cd $i echo pwd echo $ls done echo quit) | ftp ${ftpflags} ${remhost} SHAR_EOF chmod +x 'ftplist' fi # end of overwriting check if test -f 'ftpfind' then echo shar: will not over-write existing file "'ftpfind'" else cat << \SHAR_EOF > 'ftpfind' #! /bin/sh # Runs ftplist -v -r on a list of sites, filtering the output # through ftpfilter and storing the (compressed) output in # $FTPDIR/hostname.Z (older copies are aged to keep the last two around) # Sends a diff with the last copy to stdout # Mark Moraes, University of Toronto # # PING must be Mike Muuss' ping mhome=`grep moraes /etc/passwd | awk -F: '{print $6}'` PATH=$mhome/bin:/local/bin:/usr/ucb:/usr/bin:/bin export PATH PING=/local/etc/ping # !! Really should allow an option for this FTPDIR=${FTPDIR-/u/moraes/ftplists} cd $FTPDIR for i do case $i in *:*) host=`expr "$i" : "\(.*\):.*"` dir="-d "`expr "$i" : ".*:\(.*\)"` ;; *) host=$i dir= ;; esac if $PING $host 1 6 > /dev/null 2>&1 ; then if ftplist -v -r -h $host $dir | ftpfilter -l > $host; then if test -s $host ; then echo $host ; echo :::::::::: zdiff $host.Z $host agelog $host.Z 1 ; rm -f $host.Z compress $host else rm -f $host echo $host ftplist failed >&2 fi else rm -f $host echo $host ftplist or ftpfilter failed >&2 fi else echo $host unreachable >&2 fi echo :::::::::: done SHAR_EOF chmod +x 'ftpfind' fi # end of overwriting check if test -f 'ftpget' then echo shar: will not over-write existing file "'ftpget'" else cat << \SHAR_EOF > 'ftpget' #! /bin/sh # $Id: ftpget,v 1.2 91/05/07 21:15:43 moraes Exp $ # Script to perform batch anonymous ftp. Essentially converts a list of # of command line arguments into input to ftp. # Simple, and quick - written as a companion to ftplist # -h specifies the remote host (default prep.ai.mit.edu) # -d specifies the remote directory to cd to - you can provide a sequence # of -d options - they will be cd'ed to in turn. If the paths are relative, # make sure you get the sequence right. Be careful with relative paths - # there are far too many symlinks nowadays. # (default is the ftp login directory) # -v turns on the verbose option of ftp, and shows all responses from the # ftp server. # -f remotefile[:localfile] gets the remote file into localfile # -m pattern does an mget with the specified pattern. Remember to quote # shell characters. # -c does a local cd to the specified directory # For example, # ftpget -h expo.lcs.mit.edu -d contrib -f xplaces.shar:xplaces.sh \ # -d ../pub/R3/fixes -c ~/fixes -m 'fix*' # will get xplaces.shar from ~ftp/contrib on expo.lcs.mit.edu, and put it in # xplaces.sh in the current working directory, and get all fixes from # ~ftp/pub/R3/fixes and put them in the ~/fixes directory. # Obviously, the sequence of the options is important, since the equivalent # commands are executed by ftp in corresponding order # # Mark Moraes , Feb 1, 1989 # PATH=/local/bin:/usr/ucb:/usr/bin:/bin export PATH TMPFILE=/tmp/ftp.$$ SITE=`domainname`.toronto.edu usage="Usage: $0 [-h remotehost] [-d remotedirectory]... [-f remfile:localfile]... \ [-c localdirectory] [-m filepattern] [-v]" ftpflags="-i -n" verbflag= set -f # So we can use globbing in -m set x `getopt vh:d:c:m:f: $*` if [ $? != 0 ]; then echo $usage exit 1 fi shift trap 'rm -f ${TMPFILE} ; exit' 0 1 2 3 15 echo user anonymous ${USER-gnu}@${SITE} > ${TMPFILE} echo binary >> ${TMPFILE} for i in $* do case $i in -v) verbflag=-v; echo hash >> ${TMPFILE}; shift;; -h) remhost=$2; shift 2;; -d) echo cd $2 >> ${TMPFILE}; if [ x${verbflag} != x ]; then echo pwd >> ${TMPFILE}; fi; shift 2;; -c) echo lcd $2 >> ${TMPFILE}; shift 2;; -m) echo mget "$2" >> ${TMPFILE}; shift 2;; -f) f1=`expr "$2" : "\([^:]*\).*"`; f2=`expr "$2" : "[^:]*:\(.*\)"`; echo get ${f1} ${f2} >> ${TMPFILE}; shift 2;; --) shift; break;; esac done if [ $# -ne 0 ]; then echo $usage exit 2 fi if [ x${verbflag} != x ]; then ftpflags="${ftpflags} -v" fi if [ x${remhost} = x ]; then remhost=prep.ai.mit.edu fi echo quit >> ${TMPFILE} ftp ${ftpflags} ${remhost} < ${TMPFILE} rm -f ${TMPFILE} SHAR_EOF chmod +x 'ftpget' fi # end of overwriting check if test -f 'grabfiles' then echo shar: will not over-write existing file "'grabfiles'" else cat << \SHAR_EOF > 'grabfiles' #! /bin/sh # Takes the output of an ftpgrep and gets those files by generating # anonymous ftp scripts and running them. This allows one to ftpgrep # for something, extract the lines that one wants and feed them # to this script. (The extraction may be done with grep, sed, awk, # or cut and paste) # Mark Moraes, University of Toronto. cmdname=`basename $0` usage="Usage: $cmdname [-v]" verbflag= case $# in 0) ;; 1) case $1 in -v) verbflag=-v ;; *) echo $usage >&2 exit 1 ;; esac ;; *) echo $usage >&2 exit 1 ;; esac #set -e tmp=/tmp/grab$$ # Get the list of sites sites=`tee $tmp | sort -t: -u +0 -1 | sed 's/^\([^:]*\).*/\1/'` # For each site for i in $sites do egrep '^'$i $tmp | sed 's,^[^:]*:\(.*\),\1,' | tr '/' '\001' | sort | tr '\001' '/' | awk 'BEGIN { ftp="'$tmp.1'"; print "open '$i'" > ftp; print "user anonymous '$USER'" >> ftp; print "binary" >> ftp; if ("'$verbflag'" == "-v") { print "hash" >> ftp; } top = "'$i'"; print "mkdir " top; print "lcd " top >> ftp; top = top "/"; cwd = "" cwdsep = "" } /^.*\/$/ { # print "state 1: " cwd if (substr($1, 1, 1) == "/") { cwd = substr($1, 2, length($1) - 1); } else { cwd = $1 } if (substr(cwd, length(cwd), 1) == "/") { cwd = substr(cwd, 1, length(cwd) - 1); } cwdsep = "/"; print "mkdir -p " top cwd; print "mget " cwd cwdsep "*" >> ftp; next; } { n = split($1, path, "/"); file = path[n]; dir = substr($1, 1, length($1)-length(path[n])- 1); if (substr(dir, 1, 1) == "/") { dir = substr(dir, 2, length(dir) - 1); } if (dir != cwd) { cwd = dir; cwdsep = "/"; print "mkdir -p " top cwd; } print "get " cwd cwdsep path[n] >> ftp; # print "state 2: " cwd " file " path[n] next; } END {print "quit" >> ftp}' | sh $verbflag ftp $verbflag -i -n < $tmp.1 rm -f $tmp.1 done rm -f $tmp SHAR_EOF chmod +x 'grabfiles' fi # end of overwriting check if test -f 'ship' then echo shar: will not over-write existing file "'ship'" else cat << \SHAR_EOF > 'ship' #!/bin/sh # # ship filename recipient # # send a big binary over mail. Applies appropriate encoding (e.g. btoa or # rscsencode), splits into managable chunks and wraps each part for easy # reconstruction. Useful for sending compressed files or tar files or # compressed tar files. If filename is a pathname, then all leading # components will be removed from the filename that is sent. # # Jean-Francois Lamy, lamy@ai.toronto.edu, 1986-06-20 # Mark Moraes, moraes@csri.toronto.edu, 1989-07-13 # PATH=/local/bin:/usr/ucb:/usr/bin:/bin export PATH usage="ship [-btoa|-rscs|-benc|-uu] filename recipient default is bencode" case $# in [23]) ;; *) echo "$usage" 1>&2 exit 1 ;; esac tmpdir=/tmp prefix=$tmpdir/ship$$ # Put the name of the program to encode the files here - usually btoa or # uuencode, sometimes rscsencode/bencode if BITNET is being used (thanks to # IBM's use of EBCDIC. By default, we use bencode, since it is safest) # Also tag on a suffix to the file to indicate the type of encoding used, #(eg) uu for uudecode, rscs for rscsencode and btoa for btoa. This will # result in a file like foo.uu, foo.rscs or foo.btoa respectively. encode=bencode suffix=benc decode=bdecode case $1 in -bt*) encode=btoa suffix=btoa decode=atob shift ;; -be*) shift ;; -rs*) encode=rscsencode suffix=rscs decode=rscsdecode shift ;; -uu*) # Argh - uuencode just HAS to take an argument... encode="uuencode `basename $2`" suffix=uu decode=uudecode shift ;; -*) echo Invalid option: "$1" 1>&2 echo "$usage" 1>&2 exit 1 ;; *) ;; esac case $# in 2) ;; *) echo Need filename and recipient 1>&2 echo "$usage" 1>&2 exit 1 ;; esac recipient=${2} echo lucky recipient is $recipient if test ! -f $1 ; then echo Cannot find file "$1" 1>&2 exit 1 fi name=`basename ${1}`.$suffix # Just for an intelligent unpacking message. case "$decode" in uudecode) # The uu*code family are pests! uncompress=";uncompress<$1" ;; *) uncompress=" | uncompress" ;; esac case $1 in *.tar.Z) extract="$uncompress | tar xvf -" ;; *.shar.Z|*.sh.Z) extract="$uncompress | sh" ;; *.tar) extract="| tar xvf -" ;; *.sh|*.shar) extract="| sh" ;; *.Z) extract= ;; *) echo I hope you told $recipient how to unpack this 1>&2 extract="" ;; esac $encode < $1 | split -680 - $prefix. if test ! -f $prefix.aa ; then echo 'encoding of '$1' or split into '$prefix' failed' exit 1 fi howmany=`ls -1 $prefix.*| wc |sed -e "s/ *\\([0-9][0-9]*\\) .*\$/\\1/"` currentno=1 for i in $prefix.* ; do part=`expr "$i" : "$prefix\.\(..\)"` sed -e "1i\\ #! /bin/sh\\ # This is a shell archive. Unpacking this file (by passing this message\\ # through the unshar program, or deleting everything before the '#! /bin/sh'\\ # line and passing the rest through /bin/sh) will give you\\ # $name.$part (which is part $currentno of a set of $howmany)\\ # All parts must be glued together and then unpacked with\\ # cat $name.* | $decode$extract\\ sed -e 's/^X//' > $name.$part <<\\\\end_of_$name.$part" \ -e "s/^/X/" \ -e "\$a\\ end_of_$name.$part\\ " $i | mail -s "$name, part $currentno of $howmany" $recipient echo $name.$part, $currentno of $howmany /bin/rm -f $i currentno=`expr $currentno + 1` done SHAR_EOF chmod +x 'ship' fi # end of overwriting check if test -f 'unpackmail' then echo shar: will not over-write existing file "'unpackmail'" else cat << \SHAR_EOF > 'unpackmail' #! /bin/sh # Takes mail messages in from a mailbox, assumes they are the pieces # of a file that has been split and mailed with ship, and unpacks it. # DO NOT RUN THIS COMMAND ON MAIL YOU DO NOT TRUST. IT USES COMMANDS # FOUND IN THE MAIL MESSAGE, AND IS THEREFORE INSECURE. command=`grep '^# cat ' $@ | sed -e 's/^# //' -e q` case "$command" in '') echo 'Cannot unpack the mailbox automatically. Sorry.' >&2 exit 1 ;; esac echo "The command that will be used to unpack this is:" >&2 echo " $command" >&2 awk 'END {printf "Ok (y/n)? "}' < /dev/null read ans case "$ans" in [yY]*) ;; *) echo 'Quitting.' >&2;; esac sed -n ':start x s/.*/unshar << \\what_a_maroon/p x p n :copy /^From /b newmsg ${ p; s/.*/what_a_maroon/p; q; } p n b copy :newmsg x s/.*/what_a_maroon/p x b start ' $@ | sh eval "$command" remove=`expr "$command" : "cat \([^ ]*\) .*"` echo 'remove the files '"$remove"' ... files to clean up' >&2 SHAR_EOF chmod +x 'unpackmail' fi # end of overwriting check # End of shell archive exit 0