PATCH: support for nanosecond timestamps

PATCH: support for nanosecond timestamps

[Oliver Kiddle]

The following patch adds support in zsh for high resolution timestamps
on systems that support them. This affects the following:
    -N (file unread) conditional flag
    -ot (older than) and -nt (newer than) conditional flags
    o/O (ordered) glob qualifiers with a/m/c sort specifiers (including
         with - glob qualifier to act on symlinks)

Does anyone know of any other features that could make use of this? I
have grepped for mtime/atime in the source and the other references are
for checking for history/utmp/mail file updates and the stat module.

I've noticed occasionally in the past that things like (om) and -nt
haven't done as I wanted because operations are simply too fast so I'm
hoping this will be useful. Hopefully it doesn't break anyone's
scripts.

Oliver

PS. Does curses_keys.h need to be added to .cvsignore perhaps: cvs
shouldn't be listing it in the diff output below.

? Src/Modules/curses_keys.h
Index: configure.ac
===================================================================
RCS file: /cvsroot/zsh/zsh/configure.ac,v
retrieving revision 1.76
diff -u -r1.76 configure.ac
--- configure.ac	29 Oct 2007 10:30:06 -0000	1.76
+++ configure.ac	1 Nov 2007 15:15:29 -0000
@@ -977,6 +977,17 @@
   AC_DEFINE(sigset_t, unsigned int)
 fi
 
+dnl check structures for high resolution timestamps
+AC_CHECK_MEMBERS([struct stat.st_atim.tv_nsec,
+                  struct stat.st_atimespec.tv_nsec,
+                  struct stat.st_atimensec,
+                  struct stat.st_mtim.tv_nsec,
+                  struct stat.st_mtimespec.tv_nsec,
+                  struct stat.st_mtimensec,
+                  struct stat.st_ctim.tv_nsec,
+                  struct stat.st_ctimespec.tv_nsec,
+                  struct stat.st_ctimensec])
+
 dnl Check for struct timezone since some old SCO versions do not define it
 zsh_TYPE_EXISTS([
 #ifdef HAVE_SYS_TIME_H
Index: Src/cond.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/cond.c,v
retrieving revision 1.13
diff -u -r1.13 cond.c
--- Src/cond.c	24 Jul 2007 09:36:48 -0000	1.13
+++ Src/cond.c	1 Nov 2007 15:15:30 -0000
@@ -344,19 +344,39 @@
     case 'G':
 	return !((st = getstat(left)) && st->st_gid == getegid());
     case 'N':
+#if defined(GET_ST_MTIME_NSEC) && defined(GET_ST_ATIME_NSEC)
+	if (!(st = getstat(left)))
+	    return 1;
+        return (st->st_atime == st->st_mtime) ?
+        	GET_ST_ATIME_NSEC(*st) > GET_ST_MTIME_NSEC(*st) :
+        	st->st_atime > st->st_mtime;
+#else
 	return !((st = getstat(left)) && st->st_atime <= st->st_mtime);
+#endif
     case 't':
 	return !isatty(mathevali(left));
     case COND_NT:
     case COND_OT:
 	{
 	    time_t a;
+#ifdef GET_ST_MTIME_NSEC
+	    long nsecs;
+#endif
 
 	    if (!(st = getstat(left)))
 		return 1;
 	    a = st->st_mtime;
+#ifdef GET_ST_MTIME_NSEC
+	    nsecs = GET_ST_MTIME_NSEC(*st);
+#endif
 	    if (!(st = getstat(right)))
 		return 1;
+#ifdef GET_ST_MTIME_NSEC
+	    if (a == st->st_mtime) {
+                return !((ctype == COND_NT) ? nsecs > GET_ST_MTIME_NSEC(*st) :
+                        nsecs < GET_ST_MTIME_NSEC(*st));
+	    }
+#endif
 	    return !((ctype == COND_NT) ? a > st->st_mtime : a < st->st_mtime);
 	}
     case COND_EF:
Index: Src/glob.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/glob.c,v
retrieving revision 1.60
diff -u -r1.60 glob.c
--- Src/glob.c	22 Oct 2007 09:27:05 -0000	1.60
+++ Src/glob.c	1 Nov 2007 15:15:39 -0000
@@ -52,6 +52,18 @@
     long _mtime;
     long _ctime;
     long _links;
+#ifdef GET_ST_ATIME_NSEC
+    long ansec;
+    long _ansec;
+#endif
+#ifdef GET_ST_MTIME_NSEC
+    long mnsec;
+    long _mnsec;
+#endif
+#ifdef GET_ST_CTIME_NSEC
+    long cnsec;
+    long _cnsec;
+#endif
 };
 
 #define GS_NAME   1
@@ -373,6 +385,15 @@
 	    matchptr->mtime = buf.st_mtime;
 	    matchptr->ctime = buf.st_ctime;
 	    matchptr->links = buf.st_nlink;
+#ifdef GET_ST_ATIME_NSEC
+	    matchptr->ansec = GET_ST_ATIME_NSEC(buf);
+#endif
+#ifdef GET_ST_MTIME_NSEC
+	    matchptr->mnsec = GET_ST_MTIME_NSEC(buf);
+#endif
+#ifdef GET_ST_CTIME_NSEC
+	    matchptr->cnsec = GET_ST_CTIME_NSEC(buf);
+#endif
 	}
 	if (statted & 2) {
 	    matchptr->_size = buf2.st_size;
@@ -380,6 +401,15 @@
 	    matchptr->_mtime = buf2.st_mtime;
 	    matchptr->_ctime = buf2.st_ctime;
 	    matchptr->_links = buf2.st_nlink;
+#ifdef GET_ST_ATIME_NSEC
+	    matchptr->_ansec = GET_ST_ATIME_NSEC(buf);
+#endif
+#ifdef GET_ST_MTIME_NSEC
+	    matchptr->_mnsec = GET_ST_MTIME_NSEC(buf);
+#endif
+#ifdef GET_ST_CTIME_NSEC
+	    matchptr->_cnsec = GET_ST_CTIME_NSEC(buf);
+#endif
 	}
 	matchptr++;
 
@@ -885,12 +915,24 @@
 	    break;
 	case GS_ATIME:
 	    r = a->atime - b->atime;
+#ifdef GET_ST_ATIME_NSEC
+            if (!r)
+              r = a->ansec - b->ansec;
+#endif
 	    break;
 	case GS_MTIME:
 	    r = a->mtime - b->mtime;
+#ifdef GET_ST_MTIME_NSEC
+            if (!r)
+              r = a->mnsec - b->mnsec;
+#endif
 	    break;
 	case GS_CTIME:
 	    r = a->ctime - b->ctime;
+#ifdef GET_ST_CTIME_NSEC
+            if (!r)
+              r = a->cnsec - b->cnsec;
+#endif
 	    break;
 	case GS_LINKS:
 	    r = b->links - a->links;
@@ -900,12 +942,24 @@
 	    break;
 	case GS__ATIME:
 	    r = a->_atime - b->_atime;
+#ifdef GET_ST_ATIME_NSEC
+            if (!r)
+              r = a->_ansec - b->_ansec;
+#endif
 	    break;
 	case GS__MTIME:
 	    r = a->_mtime - b->_mtime;
+#ifdef GET_ST_MTIME_NSEC
+            if (!r)
+              r = a->_mnsec - b->_mnsec;
+#endif
 	    break;
 	case GS__CTIME:
 	    r = a->_ctime - b->_ctime;
+#ifdef GET_ST_CTIME_NSEC
+            if (!r)
+              r = a->_cnsec - b->_cnsec;
+#endif
 	    break;
 	case GS__LINKS:
 	    r = b->_links - a->_links;
Index: Src/system.h
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/system.h,v
retrieving revision 1.47
diff -u -r1.47 system.h
--- Src/system.h	11 Oct 2007 20:32:00 -0000	1.47
+++ Src/system.h	1 Nov 2007 15:15:40 -0000
@@ -805,3 +805,24 @@
 # define USE_GETPWUID
 #endif
 
+#ifdef HAVE_STRUCT_STAT_ST_ATIM_TV_NSEC
+# define GET_ST_ATIME_NSEC(st) (st).st_atim.tv_nsec
+#elif HAVE_STRUCT_STAT_ST_ATIMESPEC_TV_NSEC
+# define GET_ST_ATIME_NSEC(st) (st).st_atimespec.tv_nsec
+#elif HAVE_STRUCT_STAT_ST_ATIMENSEC
+# define GET_ST_ATIME_NSEC(st) (st).st_atimensec
+#endif
+#ifdef HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC
+# define GET_ST_MTIME_NSEC(st) (st).st_mtim.tv_nsec
+#elif HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC
+# define GET_ST_MTIME_NSEC(st) (st).st_mtimespec.tv_nsec
+#elif HAVE_STRUCT_STAT_ST_MTIMENSEC
+# define GET_ST_MTIME_NSEC(st) (st).st_mtimensec
+#endif
+#ifdef HAVE_STRUCT_STAT_ST_CTIM_TV_NSEC
+# define GET_ST_CTIME_NSEC(st) (st).st_ctim.tv_nsec
+#elif HAVE_STRUCT_STAT_ST_CTIMESPEC_TV_NSEC
+# define GET_ST_CTIME_NSEC(st) (st).st_ctimespec.tv_nsec
+#elif HAVE_STRUCT_STAT_ST_CTIMENSEC
+# define GET_ST_CTIME_NSEC(st) (st).st_ctimensec
+#endif

Re: PATCH: support for nanosecond timestamps

[Stephane Chazelas]

On Thu, Nov 01, 2007 at 04:40:01PM +0100, Oliver Kiddle wrote:
> The following patch adds support in zsh for high resolution timestamps
> on systems that support them. This affects the following:
>     -N (file unread) conditional flag
>     -ot (older than) and -nt (newer than) conditional flags
>     o/O (ordered) glob qualifiers with a/m/c sort specifiers (including
>          with - glob qualifier to act on symlinks)

Thanks Oliver,

that's very helpful.

> Does anyone know of any other features that could make use of this? I
> have grepped for mtime/atime in the source and the other references are
> for checking for history/utmp/mail file updates and the stat module.
[...]

What about returning floating point numbers for zstat?

Then maybe we'd need the ability to make EPOCHSECONDS a float as
well (using gettimeofday())

Cheers,
stéphane

Re: PATCH: support for nanosecond timestamps

[Oliver Kiddle]

Stephane Chazelas wrote:
> What about returning floating point numbers for zstat?

I've just looked at the documentation for stat to see how that might
work. It'd end up returning the string representation rather than a float
given the current interface. What exactly were you envisaging? It might
be more useful to have it available with the -F (format) option. The only
trouble there is that there is no standard for the letters to use in the
format specifiers.  date(1) on Linux has %N for nanoseconds. Apache uses
%q and %Q for milli and micro seconds. Do you know of any others?

Oliver

Re: PATCH: support for nanosecond timestamps

[Stephane Chazelas]

On Thu, Nov 01, 2007 at 05:37:53PM +0000, Oliver Kiddle wrote:
> Stephane Chazelas wrote:
> > What about returning floating point numbers for zstat?
> 
> I've just looked at the documentation for stat to see how that might
> work. It'd end up returning the string representation rather than a float
> given the current interface. What exactly were you envisaging? It might
> be more useful to have it available with the -F (format) option. The only
> trouble there is that there is no standard for the letters to use in the
> format specifiers.  date(1) on Linux has %N for nanoseconds. Apache uses
> %q and %Q for milli and micro seconds. Do you know of any others?
[...]

%N is also used by ksh93:

printf '%(%N)T\n'

And there's %.s or %.# to have sec.nanosec

Cheers,
Stéphane

Re: PATCH: support for nanosecond timestamps

[Peter Stephenson]

On Thu, 01 Nov 2007 16:40:01 +0100
Oliver Kiddle <okiddle@yahoo.co.uk> wrote:
> PS. Does curses_keys.h need to be added to .cvsignore perhaps: cvs
> shouldn't be listing it in the diff output below.

Yes, I've done that.

I've also added a note on high-resolution timestamps to NEWS before I
forget, though NEWS is fairly well out of date anyway.

-- 
Peter Stephenson <p.w.stephenson@ntlworld.com>
Web page now at http://homepage.ntlworld.com/p.w.stephenson/

Re: PATCH: support for nanosecond timestamps

[Peter Stephenson]

On Thu, 01 Nov 2007 17:37:53 +0000
Oliver Kiddle <okiddle@yahoo.co.uk> wrote:
> Stephane Chazelas wrote:
> > What about returning floating point numbers for zstat?
> 
> I've just looked at the documentation for stat to see how that might
> work. It'd end up returning the string representation rather than a float
> given the current interface. What exactly were you envisaging? It might
> be more useful to have it available with the -F (format) option. The only
> trouble there is that there is no standard for the letters to use in the
> format specifiers.  date(1) on Linux has %N for nanoseconds.

%N seems reasonable, although I'm wondering if we might actually be
better off with a specification for floating point seconds; we could
then use the same thing in zsh/datetime where we only get microsecond
resolution from gettimeofday().  Either way, it would probably be good
to add this to ztrftime() and expand the interface to include a
(possibly NULL) pointer to whatever gives the extra resolution.  Would
it be too horrible to allow formats like "%.9s"?  We'd have to restrict
it to %S and %s which are the only two where it makes sense, and we
would have to decide what to do about E and O modifiers and glibc flag
additions; simply passing them straight through and not allowing a
fractional second in such cases would probably be good enough.

-- 
Peter Stephenson <p.w.stephenson@ntlworld.com>
Web page now at http://homepage.ntlworld.com/p.w.stephenson/

Re: PATCH: support for nanosecond timestamps

[Oliver Kiddle]

Peter Stephenson wrote:
> %N seems reasonable, although I'm wondering if we might actually be
> better off with a specification for floating point seconds; we could
> then use the same thing in zsh/datetime where we only get microsecond
> resolution from gettimeofday().  Either way, it would probably be good
> to add this to ztrftime() and expand the interface to include a
> (possibly NULL) pointer to whatever gives the extra resolution.  Would
> it be too horrible to allow formats like "%.9s"?  We'd have to restrict
> it to %S and %s which are the only two where it makes sense, and we
> would have to decide what to do about E and O modifiers and glibc flag
> additions; simply passing them straight through and not allowing a
> fractional second in such cases would probably be good enough.

How about the following as an initial step? This only adds %N. I've
expanded the ztrftime interface with a long for nanoseconds. That seemed
as good as any pointer given that 0 is going to be the default value.
The only cases where I've not made it pass 0 is stat and prompt
expansion (where we have gettimeofday). So the datetime module is
unchanged.

Formats like %.9s make a certain amount of logical sense though you
might expect that to output the seconds too, e.g.: 23.562827621
It might be better to allow %6N and %3N given that %N is a non-standard
extension already.

The E and 0 modifiers don't seem to work for me, are they supposed to?

Oliver

Index: Src/prompt.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/prompt.c,v
retrieving revision 1.43
diff -u -r1.43 prompt.c
--- Src/prompt.c	22 Oct 2007 10:22:19 -0000	1.43
+++ Src/prompt.c	5 Nov 2007 14:03:17 -0000
@@ -210,6 +210,8 @@
     char *ss, *hostnam;
     int t0, arg, test, sep, j, numjobs;
     struct tm *tm;
+    struct timeval timev;
+    struct timezone dummy;
     time_t timet;
     Nameddir nd;
 
@@ -529,8 +531,8 @@
 			tmfmt = "%l:%M%p";
 			break;
 		    }
-		    timet = time(NULL);
-		    tm = localtime(&timet);
+		    gettimeofday(&timev, &dummy);
+		    tm = localtime((const time_t*) &timev.tv_sec);
 		    /*
 		     * Hack because strftime won't say how
 		     * much space it actually needs.  Try to add it
@@ -540,7 +542,8 @@
 		     */
 		    for(j = 0, t0 = strlen(tmfmt)*8; j < 3; j++, t0*=2) {
 			addbufspc(t0);
-			if (ztrftime(bp, t0, tmfmt, tm) >= 0)
+			if (ztrftime(bp, t0, tmfmt, tm, 1000 * timev.tv_usec)
+                        	>= 0)
 			    break;
 		    }
 		    /* There is enough room for this because addbufspc(t0)
Index: Src/utils.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/utils.c,v
retrieving revision 1.169
diff -u -r1.169 utils.c
--- Src/utils.c	31 Oct 2007 06:22:57 -0000	1.169
+++ Src/utils.c	5 Nov 2007 14:03:18 -0000
@@ -2357,7 +2357,7 @@
 
 /**/
 mod_export int
-ztrftime(char *buf, int bufsize, char *fmt, struct tm *tm)
+ztrftime(char *buf, int bufsize, char *fmt, struct tm *tm, long nsecs)
 {
     int hr12, decr;
 #ifndef HAVE_STRFTIME
@@ -2425,6 +2425,12 @@
 		*buf++ = '0' + tm->tm_min / 10;
 		*buf++ = '0' + tm->tm_min % 10;
 		break;
+            case 'N':
+		if (ztrftimebuf(&bufsize, 9))
+		    return 0;
+                sprintf(buf, "%09ld", nsecs);
+                buf += 9;
+		break;
 	    case 'S':
 		*buf++ = '0' + tm->tm_sec / 10;
 		*buf++ = '0' + tm->tm_sec % 10;
Index: Src/watch.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/watch.c,v
retrieving revision 1.5
diff -u -r1.5 watch.c
--- Src/watch.c	4 Apr 2005 09:59:05 -0000	1.5
+++ Src/watch.c	5 Nov 2007 14:03:18 -0000
@@ -330,7 +330,7 @@
 		    }
 		    timet = getlogtime(u, inout);
 		    tm = localtime(&timet);
-		    ztrftime(buf, 40, fm2, tm);
+		    ztrftime(buf, 40, fm2, tm, 0);
 		    printf("%s", (*buf == ' ') ? buf + 1 : buf);
 		    break;
 		case '%':
Index: Src/Builtins/sched.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Builtins/sched.c,v
retrieving revision 1.13
diff -u -r1.13 sched.c
--- Src/Builtins/sched.c	6 Jul 2007 21:52:40 -0000	1.13
+++ Src/Builtins/sched.c	5 Nov 2007 14:03:18 -0000
@@ -211,7 +211,7 @@
 
 	    t = sch->time;
 	    tmp = localtime(&t);
-	    ztrftime(tbuf, 20, "%a %b %e %k:%M:%S", tmp);
+	    ztrftime(tbuf, 20, "%a %b %e %k:%M:%S", tmp, 0);
 	    if (sch->flags & SCHEDFLAG_TRASH_ZLE)
 		flagstr = "-o ";
 	    else
Index: Src/Modules/datetime.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Modules/datetime.c,v
retrieving revision 1.17
diff -u -r1.17 datetime.c
--- Src/Modules/datetime.c	6 Jul 2007 21:52:40 -0000	1.17
+++ Src/Modules/datetime.c	5 Nov 2007 14:03:18 -0000
@@ -125,7 +125,7 @@
     buffer = zalloc(bufsize);
 
     for (x=0; x < 4; x++) {
-        if (ztrftime(buffer, bufsize, argv[0], t) >= 0)
+        if (ztrftime(buffer, bufsize, argv[0], t, 0) >= 0)
 	    break;
 	buffer = zrealloc(buffer, bufsize *= 2);
     }
Index: Src/Modules/stat.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Modules/stat.c,v
retrieving revision 1.15
diff -u -r1.15 stat.c
--- Src/Modules/stat.c	6 Jul 2007 21:52:40 -0000	1.15
+++ Src/Modules/stat.c	5 Nov 2007 14:03:18 -0000
@@ -188,7 +188,7 @@
 
 /**/
 static void
-stattimeprint(time_t tim, char *outbuf, int flags)
+stattimeprint(time_t tim, long nsecs, char *outbuf, int flags)
 {
     if (flags & STF_RAW) {
 	sprintf(outbuf, "%ld", (unsigned long)tim);
@@ -198,7 +198,7 @@
     if (flags & STF_STRING) {
 	char *oend = outbuf + strlen(outbuf);
 	ztrftime(oend, 40, timefmt, (flags & STF_GMT) ? gmtime(&tim) :
-		 localtime(&tim));
+		 localtime(&tim), nsecs);
 	if (flags & STF_RAW)
 	    strcat(oend, ")");
     }
@@ -288,15 +288,27 @@
 	break;
 
     case ST_ATIM:
-	stattimeprint(sbuf->st_atime, optr, flags);
+#ifdef GET_ST_ATIME_NSEC
+	stattimeprint(sbuf->st_atime, GET_ST_ATIME_NSEC(*sbuf), optr, flags);
+#else
+	stattimeprint(sbuf->st_atime, 0, optr, flags);
+#endif
 	break;
 
     case ST_MTIM:
-	stattimeprint(sbuf->st_mtime, optr, flags);
+#ifdef GET_ST_ATIME_NSEC
+	stattimeprint(sbuf->st_atime, GET_ST_MTIME_NSEC(*sbuf), optr, flags);
+#else
+	stattimeprint(sbuf->st_mtime, 0, optr, flags);
+#endif
 	break;
 
     case ST_CTIM:
-	stattimeprint(sbuf->st_ctime, optr, flags);
+#ifdef GET_ST_ATIME_NSEC
+	stattimeprint(sbuf->st_atime, GET_ST_CTIME_NSEC(*sbuf), optr, flags);
+#else
+	stattimeprint(sbuf->st_ctime, 0, optr, flags);
+#endif
 	break;
 
     case ST_BLKSIZE:

Re: PATCH: support for nanosecond timestamps

[Stephane Chazelas]

On Mon, Nov 05, 2007 at 02:45:50PM +0000, Oliver Kiddle wrote:
[...]
> Formats like %.9s make a certain amount of logical sense though you
> might expect that to output the seconds too, e.g.: 23.562827621
> It might be better to allow %6N and %3N given that %N is a non-standard
> extension already.
[...]

Again, ksh93:

$ ksh -c 'printf "%(%.s)T\n"'
1194276214.312372000
$ ksh -c 'printf "%(%.9s)T\n"'
1194276216.337172000
$ ksh -c 'printf "%(%.3s)T\n"'
1194276220.963
$ ksh -c 'printf "%(%s)T\n"'
1194276238
$ ksh -c 'printf "%(%N)T\n"'
001528000
$ ksh -c 'printf "%(%3N)T\n"'
576
$ ksh -c 'printf "%(%6N)T\n"'
825647

For %<x>N, the question is: "should they be zero-padded?"

-- 
Stéphane

Re: PATCH: support for nanosecond timestamps

[Peter Stephenson]

Oliver Kiddle wrote:
> Formats like %.9s make a certain amount of logical sense though you
> might expect that to output the seconds too, e.g.: 23.562827621

Yes, that's what I meant.

> The E and 0 modifiers don't seem to work for me, are they supposed to?

Quite likely no one ever tried; I certainly haven't.  It may be we don't
know they're modifiers and so don't pass them on properly (in which case
it will presumably be the same with the glibc extensions).

-- 
Peter Stephenson <pws@csr.com>                  Software Engineer
CSR PLC, Churchill House, Cambridge Business Park, Cowley Road
Cambridge, CB4 0WZ, UK                          Tel: +44 (0)1223 692070