new module

new module

[Zefram]

-----BEGIN PGP SIGNED MESSAGE-----

Here's a useful module.  It implements the most basic file-manipulation
commands.  (I could have used this last week, when I filled up the
process table and needed to remove a file in order to unfill it.)

Zoltan, why does addbuiltin() not allow the funcid and defopts members of
the builtin structure to be set?  They would be helpful in implementing
mv in this module.

 -zefram

      Index: Src/Modules/Makefile.in
      ===================================================================
      RCS file: /home/zefram/usr/cvsroot/zsh/Src/Modules/Makefile.in,v
      retrieving revision 1.3
      diff -c -r1.3 Makefile.in
      *** Src/Modules/Makefile.in	1996/12/23 02:34:09	1.3
      --- Src/Modules/Makefile.in	1996/12/23 02:40:44
      ***************
      *** 86,95 ****
        	$(SED) -n -f $(srcdir)/../makepro.sed $< > $@
        
        # generated prototypes
      ! PROTO = example.pro
        
        # target modules
      ! MODULES = example.so
        
        # ========= DEPENDENCIES FOR BUILDING ==========
        
      --- 86,95 ----
        	$(SED) -n -f $(srcdir)/../makepro.sed $< > $@
        
        # generated prototypes
      ! PROTO = example.pro files.pro
        
        # target modules
      ! MODULES = example.so files.so
        
        # ========= DEPENDENCIES FOR BUILDING ==========
        
      Index: Src/Modules/files.c
      ===================================================================
      RCS file: files.c
      diff -N files.c
      *** ..................../dev/null	Thu Sep  7 20:04:30 1995
      --- Src/Modules/files.c	Mon Dec 23 06:07:44 1996
      ***************
      *** 0 ****
      --- 1,466 ----
      + /*
      +  * $Id$
      +  *
      +  * files.c - file operation builtins
      +  *
      +  * This file is part of zsh, the Z shell.
      +  *
      +  * Copyright (c) 1996 Andrew Main
      +  * All rights reserved.
      +  *
      +  * Permission is hereby granted, without written agreement and without
      +  * license or royalty fees, to use, copy, modify, and distribute this
      +  * software and its documentation for any purpose, provided that the
      +  * above copyright notice and the following two paragraphs appear in
      +  * all copies of this software.
      +  *
      +  * In no event shall Andrew Main or the Zsh Development Group be liable
      +  * to any party for direct, indirect, special, incidental, or consequential
      +  * damages arising out of the use of this software and its documentation,
      +  * even if Andrew Main and the Zsh Development Group have been advised of
      +  * the possibility of such damage.
      +  *
      +  * Andrew Main and the Zsh Development Group specifically disclaim any
      +  * warranties, including, but not limited to, the implied warranties of
      +  * merchantability and fitness for a particular purpose.  The software
      +  * provided hereunder is on an "as is" basis, and Andrew Main and the
      +  * Zsh Development Group have no obligation to provide maintenance,
      +  * support, updates, enhancements, or modifications.
      +  *
      +  */
      + 
      + #include "zsh.h"
      + 
      + typedef int (*MoveFunc) _((char const *, char const *));
      + 
      + #include "files.pro"
      + 
      + static char buf[PATH_MAX * 2];
      + static char rbuf[PATH_MAX];
      + 
      + /**/
      + static int
      + ask(void)
      + {
      +     int a = getchar(), c;
      +     for(c = a; c != EOF && c != '\n'; )
      + 	c = getchar();
      +     return a == 'y' || a == 'Y';
      + }
      + 
      + /* sync builtin */
      + 
      + /**/
      + static int
      + bin_sync(char *nam, char **args, char *ops, int func)
      + {
      +     sync();
      +     return 0;
      + }
      + 
      + /* mkdir builtin */
      + 
      + /**/
      + static int
      + bin_mkdir(char *nam, char **args, char *ops, int func)
      + {
      +     mode_t oumask = umask(0);
      +     mode_t mode = 0777 & ~oumask;
      +     int err = 0;
      + 
      +     umask(oumask);
      +     if(ops['m']) {
      + 	char *str = *args++, *ptr;
      + 
      + 	if(!*args) {
      + 	    zwarnnam(nam, "not enough arguments", NULL, 0);
      + 	    return 1;
      + 	}
      + 	mode = zstrtol(str, &ptr, 8);
      + 	if(!*str || *ptr) {
      + 	    zwarnnam(nam, "invalid mode `%s'", str, 0);
      + 	    return 1;
      + 	}
      +     }
      +     for(; *args; args++) {
      + 	char *ptr = strchr(*args, 0);
      + 
      + 	while(ptr > *args + (**args == '/') && *--ptr == '/')
      + 	    *ptr = 0;
      + 	if(ztrlen(*args) > PATH_MAX - 1) {
      + 	    zwarnnam(nam, "%s: %e", *args, ENAMETOOLONG);
      + 	    err = 1;
      + 	    continue;
      + 	}
      + 	if(ops['p']) {
      + 	    char *ptr = *args;
      + 
      + 	    for(;;) {
      + 		while(*ptr == '/')
      + 		    ptr++;
      + 		while(*ptr && *ptr != '/')
      + 		    ptr++;
      + 		if(!*ptr) {
      + 		    err |= domkdir(nam, *args, mode, 1);
      + 		    break;
      + 		} else {
      + 		    int e;
      + 
      + 		    *ptr = 0;
      + 		    e = domkdir(nam, *args, mode | 0300, 1);
      + 		    if(e) {
      + 			err = 1;
      + 			break;
      + 		    }
      + 		    *ptr = '/';
      + 		}
      + 	    }
      + 	} else
      + 	    err |= domkdir(nam, *args, mode, 0);
      +     }
      +     return err;
      + }
      + 
      + /**/
      + static int
      + domkdir(char *nam, char *path, mode_t mode, int p)
      + {
      +     int err;
      +     mode_t oumask;
      +     char const *rpath = unmeta(path);
      + 
      +     if(p) {
      + 	struct stat st;
      + 
      + 	if(!stat(rpath, &st) && S_ISDIR(st.st_mode))
      + 	    return 0;
      +     }
      +     oumask = umask(0);
      +     err = mkdir(path, mode) ? errno : 0;
      +     umask(oumask);
      +     if(!err)
      + 	return 0;
      +     zwarnnam(nam, "cannot make directory `%s': %e", path, err);
      +     return 1;
      + }
      + 
      + /* rmdir builtin */
      + 
      + /**/
      + static int
      + bin_rmdir(char *nam, char **args, char *ops, int func)
      + {
      +     int err = 0;
      + 
      +     for(; *args; args++) {
      + 	char *rpath = unmeta(*args);
      + 
      + 	if(!rpath) {
      + 	    zwarnnam(nam, "%s: %e", *args, ENAMETOOLONG);
      + 	    err = 1;
      + 	} else if(rmdir(rpath)) {
      + 	    zwarnnam(nam, "cannot remove directory `%s': %e", *args, errno);
      + 	    err = 1;
      + 	}
      +     }
      +     return err;
      + }
      + 
      + /* ln and mv builtins */
      + 
      + #define BIN_LN 0
      + #define BIN_MV 1
      + 
      + #define MV_NODIRS (1<<0)
      + #define MV_FORCE  (1<<1)
      + #define MV_INTER  (1<<2)
      + #define MV_ASKNW  (1<<3)
      + 
      + /* bin_ln actually does three related jobs: hard linking, symbolic *
      +  * linking, and renaming.  If called as mv it renames, otherwise   *
      +  * it looks at the -s option.  If hard linking, it will refuse to  *
      +  * attempt linking to a directory unless the -d option is given.   */
      + 
      + /**/
      + static int
      + bin_ln(char *nam, char **args, char *ops, int func)
      + {
      +     MoveFunc move;
      +     int flags, space, err = 0;
      +     char **a, *ptr, *rp;
      +     struct stat st;
      + 
      +     if(func == BIN_MV) {
      + 	move = rename;
      + 	flags = ops['f'] ? 0 : MV_ASKNW;
      +     } else {
      + 	flags = ops['f'] ? MV_FORCE : 0;
      + #ifdef HAVE_LSTAT
      + 	if(ops['s'])
      + 	    move = symlink;
      + 	else
      + #endif
      + 	     {
      + 	    move = link;
      + 	    if(!ops['d'])
      + 		flags |= MV_NODIRS;
      + 	}
      +     }
      +     if(ops['i'])
      + 	flags |= MV_INTER;
      +     for(a = args; a[1]; a++) ;
      +     if(a != args) {
      + 	rp = unmeta(*a);
      + 	if(rp && !stat(rp, &st) && S_ISDIR(st.st_mode))
      + 	    goto havedir;
      +     }
      +     if(a > args+1) {
      + 	zwarnnam(nam, "last of many arguments must be a directory", NULL, 0);
      + 	return 1;
      +     }
      +     if(!args[1]) {
      + 	ptr = strrchr(args[0], '/');
      + 	if(ptr)
      + 	    args[1] = ptr+1;
      + 	else
      + 	    args[1] = args[0];
      +     }
      +     return domove(nam, move, args[0], args[1], flags);
      +     havedir:
      +     strcpy(buf, *a);
      +     *a = NULL;
      +     space = PATH_MAX - 2 - ztrlen(buf);
      +     rp = strchr(buf, 0);
      +     *rp++ = '/';
      +     for(; *args; args++) {
      + 	if(ztrlen(*args) > PATH_MAX - 1) {
      + 	    zwarnnam(nam, "%s: %e", *args, ENAMETOOLONG);
      + 	    err = 1;
      + 	    continue;
      + 	}
      + 	ptr = strrchr(*args, '/');
      + 	if(ptr)
      + 	    ptr++;
      + 	else
      + 	    ptr = *args;
      + 	if(ztrlen(ptr) > space) {
      + 	    zwarnnam(nam, "%s: %e", ptr, ENAMETOOLONG);
      + 	    err = 1;
      + 	    continue;
      + 	}
      + 	strcpy(rp, ptr);
      + 	err |= domove(nam, move, *args, buf, flags);
      +     }
      +     return err;
      + }
      + 
      + /**/
      + static int
      + domove(char *nam, MoveFunc move, char *p, char *q, int flags)
      + {
      +     struct stat st;
      +     char *qbuf;
      +     strcpy(rbuf, unmeta(p));
      +     qbuf = unmeta(q);
      +     /***/fprintf(stderr, "`%s' -> `%s'\n", rbuf, qbuf);
      +     if(flags & MV_NODIRS) {
      + 	errno = EISDIR;
      + 	if(lstat(rbuf, &st) || S_ISDIR(st.st_mode)) {
      + 	    zwarnnam(nam, "%s: %e", p, errno);
      + 	    return 1;
      + 	}
      +     }
      +     if(!lstat(qbuf, &st)) {
      + 	if(S_ISDIR(st.st_mode)) {
      + 	    zwarnnam(nam, "%s: cannot overwrite directory", q, 0);
      + 	    return 1;
      + 	} else if(flags & MV_INTER) {
      + 	    nicezputs(nam, stderr);
      + 	    fputs(": replace `", stderr);
      + 	    nicezputs(q, stderr);
      + 	    fputs("'? ", stderr);
      + 	    fflush(stderr);
      + 	    if(!ask())
      + 		return 0;
      + 	} else if((flags & MV_ASKNW) & access(qbuf, W_OK)) {
      + 	    nicezputs(nam, stderr);
      + 	    fputs(": replace `", stderr);
      + 	    nicezputs(q, stderr);
      + 	    fprintf(stderr, "', overriding mode %04o? ", st.st_mode & 07777);
      + 	    fflush(stderr);
      + 	    if(!ask())
      + 		return 0;
      + 	}
      + 	if(flags & MV_FORCE)
      + 	    unlink(qbuf);
      +     }
      +     if(move(rbuf, qbuf)) {
      + 	zwarnnam(nam, "%s: %e", p, errno);
      + 	return 1;
      +     }
      +     return 0;
      + }
      + 
      + /**/
      + static int
      + bin_mv(char *nam, char **args, char *ops, int func)
      + {
      +     return bin_ln(nam, args, ops, BIN_MV);
      + }
      + 
      + /* rm builtin */
      + 
      + /**/
      + static int
      + bin_rm(char *nam, char **args, char *ops, int func)
      + {
      +     int err = 0;
      + 
      +     for(; *args; args++)
      + 	err |= dorm(nam, *args, ops);
      +     return ops['f'] ? 0 : err;
      + }
      + 
      + /**/
      + static int
      + dorm(char *nam, char *arg, char *ops)
      + {
      +     struct stat st;
      +     char *rp = unmeta(arg);
      + 
      +     if(!rp) {
      + 	if(!ops['f'])
      + 	    zwarnnam(nam, "%s: %e", arg, ENAMETOOLONG);
      + 	return 1;
      +     }
      +     if((!ops['d'] || !ops['f']) && !stat(rp, &st)) {
      + 	if(!ops['d'] && S_ISDIR(st.st_mode)) {
      + 	    if(ops['r']) {
      + 		DIR *d;
      + 		char *pos, *fn;
      + 		int err = 0, space;
      + 		d = opendir(rp);
      + 		if(!d) {
      + 		    if(!ops['f'])
      + 			zwarnnam(nam, "%s: %e", arg, errno);
      + 		    return 1;
      + 		}
      + 		if(arg != buf)
      + 		    strcpy(buf, arg);
      + 		space = PATH_MAX - 2 - ztrlen(buf);
      + 		pos = strchr(buf, 0);
      + 		*pos++ = '/';
      + 		while((fn = zreaddir(d, 1))) {
      + 		    if(ztrlen(fn) > space) {
      + 			pos[-1] = 0;
      + 			zwarnnam(nam, "%s: %e", buf, ENAMETOOLONG);
      + 			err = 1;
      + 			continue;
      + 		    }
      + 		    strcpy(pos, fn);
      + 		    err |= dorm(nam, buf, ops);
      + 		}
      + 		closedir(d);
      + 		pos[-1] = 0;
      + 		if(!ops['f'] && ops['i']) {
      + 		    nicezputs(nam, stderr);
      + 		    fputs(": remove `", stderr);
      + 		    nicezputs(buf, stderr);
      + 		    fputs("'? ", stderr);
      + 		    fflush(stderr);
      + 		    if(!ask())
      + 			return err;
      + 		}
      + 		if(!rmdir(unmeta(buf)))
      + 		    return err;
      + 		if(!ops['f'])
      + 		    zwarnnam(nam, "%s: %e", buf, errno);
      + 		return 1;
      + 	    }
      + 	    if(!ops['f'])
      + 		zwarnnam(nam, "%s: %e", arg, EISDIR);
      + 	    return 1;
      + 	}
      + 	if(!ops['f'] && ops['i']) {
      + 	    nicezputs(nam, stderr);
      + 	    fputs(": remove `", stderr);
      + 	    nicezputs(arg, stderr);
      + 	    fputs("'? ", stderr);
      + 	    fflush(stderr);
      + 	    if(!ask())
      + 		return 0;
      + 	} else if(!ops['f'] && access(rp, W_OK)) {
      + 	    nicezputs(nam, stderr);
      + 	    fputs(": remove `", stderr);
      + 	    nicezputs(arg, stderr);
      + 	    fprintf(stderr, "', overriding mode %04o? ", st.st_mode & 07777);
      + 	    fflush(stderr);
      + 	    if(!ask())
      + 		return 0;
      + 	}
      +     }
      +     if(!unlink(rp))
      + 	return 0;
      +     if(!ops['f'])
      + 	zwarnnam(nam, "%s: %e", arg, errno);
      +     return 1;
      + }
      + 
      + /* module paraphernalia */
      + 
      + static struct bin {
      +     char *name;
      +     int binf;
      +     HandlerFunc func;
      +     int min, max;
      +     char *optstr;
      +     int added;
      + } bins[] = {
      + #ifdef HAVE_LSTAT
      +     { "ln",    0, bin_ln,    1, -1, "dfis", 0 },
      + #else
      +     { "ln",    0, bin_ln,    1, -1, "dfi",  0 },
      + #endif
      +     { "mkdir", 0, bin_mkdir, 1, -1, "pm",   0 },
      +     { "mv",    0, bin_mv,    2, -1, "fi",   0 },
      +     { "rm",    0, bin_rm,    1, -1, "dfir", 0 },
      +     { "rmdir", 0, bin_rmdir, 1, -1, NULL,   0 },
      +     { "sync",  0, bin_sync,  0,  0, NULL,   0 },
      +     { NULL, 0, NULL, 0, 0, NULL, 0 }
      + };
      + 
      + /**/
      + int
      + boot_files(Module m)
      + {
      +     int added = 0;
      +     struct bin *bin;
      + 
      +     for(bin = bins; bin->name; bin++) {
      + 	if (!bin->added && addbuiltin(bin->name, bin->binf, bin->func,
      + 	    bin->min, bin->max, bin->optstr))
      + 	    zwarnnam(m->nam, "name clash when adding builtin `%s'",
      + 		bin->name, 0);
      + 	else
      + 	    added = bin->added = 1;
      +     }
      +     return !added;
      + }
      + 
      + #ifdef MODULE
      + 
      + /**/
      + int
      + cleanup_files(Module m)
      + {
      +     int fail;
      +     struct bin *bin;
      + 
      +     for(bin = bins; bin->name; bin++)
      + 	if (bin->added && deletebuiltin(bin->name))
      + 	    fail = 1;
      + 	else
      + 	    bin->added = 0;
      +     return fail;
      + }
      + #endif

-----BEGIN PGP SIGNATURE-----
Version: 2.6.2

iQCVAwUBMr7DD3D/+HJTpU/hAQEvbAP+MCGdCfA11nu9vmJXk6wLQbCpUVnTm8k4
dGG8FW/CX+CQzJ48qGTRcFeG/c9cMlC6foBh3Lg4IeONFY3gs3aLOr2LBpLp9ZXz
XV/xJAmN2cShm48KG+zaMwGs6ZCbwr5k797SscZ0ZD+cBrX38WM3QP1NY6ByaTMt
awSnQsfUgYM=
=9Scr
-----END PGP SIGNATURE-----

Re: new module

[Zoltan Hidvegi]

> Here's a useful module.  It implements the most basic file-manipulation
> commands.  (I could have used this last week, when I filled up the
> process table and needed to remove a file in order to unfill it.)

It's really nice, the patch below fixes a little bug in it.

> Zoltan, why does addbuiltin() not allow the funcid and defopts members of
> the builtin structure to be set?  They would be helpful in implementing
> mv in this module.

Only because I did not see its usefullness here and I wanted a slimple
interface.  It may be added of course.

Zoltan


*** Src/Modules/files.c~	Tue Dec 24 02:33:06 1996
--- Src/Modules/files.c	Tue Dec 24 03:45:10 1996
***************
*** 453,459 ****
  int
  cleanup_files(Module m)
  {
!     int fail;
      struct bin *bin;
  
      for(bin = bins; bin->name; bin++)
--- 453,459 ----
  int
  cleanup_files(Module m)
  {
!     int fail = 0;
      struct bin *bin;
  
      for(bin = bins; bin->name; bin++)

Re: new module

[Zoltan Hidvegi]

Here are some pieces from Zefram's files.c:

>       + static char buf[PATH_MAX * 2];
>       + static char rbuf[PATH_MAX];
[...]
>       + dorm(char *nam, char *arg, char *ops)
[...]
>       + 		if(arg != buf)
>       + 		    strcpy(buf, arg);

You see I do not like this kind of code.  It works but it makes the code
more difficult to maintain.  buf is a static variable and sometimes arg and
buf is the same.  It is very easy to forget about that and it can cause you
headaches to track down a problem caused by this.  And it is almost
impossible to debug this if you call an other function which also happens
to use buf.  In files.c I do not see what's against using automatic
variable buffers.

There is one more argument against static buffers besides the clarity of
the code.  Zsh code must be as re-entrant as possible as a signal handler
can be called at any time.  E.g. while you busily doing an rm -rf foo a
signal may come and it may call an other rm (that probably violates POSIX
btw.).

I know that static buffers are used in some other places.  The result of
unmeta is in a static buffer if the string contained a metafied character.
But in other parts of the code this static buffer used immediately after
the unmeta call.

>       + 		*pos++ = '/';
>       + 		while((fn = zreaddir(d, 1))) {
>       + 		    if(ztrlen(fn) > space) {
>       + 			pos[-1] = 0;
>       + 			zwarnnam(nam, "%s: %e", buf, ENAMETOOLONG);
>       + 			err = 1;
>       + 			continue;
>       + 		    }
>       + 		    strcpy(pos, fn);

There are two problems with this code.  First it cannot remove a directory
hierarchy if it has too deep subdirectories.  That's quite bad when you use
it to clean up directories.  A user who wants to save his files from
automatic deletion can hide it deeply in a subdirectory.

The other problem if that when such an rm runs from root's cron ro clean up
/tmp it can be exploited to remove anything on the filesystem.  Any
component of a long filename can be replaced with a symlink while rm is
working which can be used to delete any tree on the system.

The right way to do that is to change into the directory we want to remove,
and stat . to make sure that the chdir did no go over a symlink.

Note that it is a more general problem: find ... -exec rm {} \; scripts are
always dangerous and can be exploited using some symlink trickery.
Therefore an option to rm which disables symlinks would be very useful
(which would allow rm -l symlink but not rm -l symlink/file).  If
implemented correctly, rm -lf <some_complicated_zsh_glob_pattern> would be
a safe operation.  Sine rm is a builtin long argument list is no longer a
problem.

E.g. when you type rm foo/bar the right procedure is:

lstat(foo), if a real directory, chpwd(foo), stat(.) and compare with the
previous lstat(foo), if not the same fail.  And now you can delete bar.

And to rm /foo/bar you have to chpwd(/) first.
Of couse a shell-builtin implementation should save pwd first and go back
after the operation is completed.

Btw. I've just noticed that rm -r in files.c uses stat() instead of lstat()
hence it follows symlinks happily :-(.

Zoltan

Re: new module

[Zefram]

Zoltan Hidvegi wrote:
>There are two problems with this code.  First it cannot remove a directory
>hierarchy if it has too deep subdirectories.  That's quite bad when you use
>it to clean up directories.  A user who wants to save his files from
>automatic deletion can hide it deeply in a subdirectory.

True.  This ought to be avoided.

>Btw. I've just noticed that rm -r in files.c uses stat() instead of lstat()
>hence it follows symlinks happily :-(.

Oops.  That was not how I intended it.  I did consider that particular
problem, and other related cases, and deliberately chose lstat() over
stat() in several places.  This I think needs to be fixed immediately.

I'll fix the other problems after the next beta, unless of course you
get to them first.

-zefram

Re: new module

[Zoltan Hidvegi]

Zefram wrote about hist new files module:
> I'll fix the other problems after the next beta, unless of course you
> get to them first.

I'd like to include a fixed version into the next beta.  The files module
should also handle interrupt signals somehow (there is no way to interrupt
it at the moment).

I did not plan to fix files.c, I hoped you'll do it.  I'd like to
concentrate on glob.c now, fix some bugs there and add the ${.../.../...}
substitution syntax, which was implemented in ksh long ago and it can now
be found in bash-2.0.  This feature would really make sed unnecessary in
zsh scripts.

A ksh glob parser and a regexp parser would also be nice.

Zoltan