Re: [9fans] ideas for helpful system io functions

[Mechiel Lukkien <mechiel@xs4all.nl>]

[-- Attachment #1: Type: text/plain, Size: 2499 bytes --]

On Sat, Dec 05, 2009 at 08:24:45AM -1000, Tim Newsham wrote:
> ps. if you wanted to hide this ugliness of passing a buffer and
> fd to a child process instead of just passing an fd, you could
> still solve it in userland without a syscall.  Write a library
> that does buffered IO.  Include unget() if you like.  Write the
> library in a way that you can initialize it after a fork/exec
> to pick up state from the parent (ie. by taking two fds,
> reading the buffer from the first, and continuing on with the
> 2nd when it is exhausted).
>
> Is there much benefit in doing this in the kernel instead?

it's all library code, and it loses the "everything is a file
(descriptor)" advantage.  you cannot pass that library state to another
program.  you could if the state was a file descriptor.

for inferno i wrote an http client library that turns a request into an
fd to read the data from.  that fd has http chunking,gzip,ssl peeled
off.  now i can pass the fd with the http response to other programs,
do buffered i/o on it, etc.  this is implemented in user-space btw,
with inferno's sys->file2chan (as opposed to pipes, you can do error
message propagation over file2chan's).

since file descriptors are so essential, it may help to have "tools"
to use them.  yesterday evening i hacked up devbuf.c and devjoin.c
after reading this thread.   both offer a file "new".  for devbuf.c
you can write data to it, then later consume it (yes, you could just
use a pipe instead).  for devjoin.c, you can write fd numbers (of open
files) to register an fd, then later reads will get data from the first
registered file, when that returns 0 it continues on the next, and so on.
so fd's can be chained for reading (not writing).  i know this "join"
functionality is different from what sam originally described.

i've attached devbuf.c and devjoin.c, as example (for inferno).
they have bugs (don't assign qid.path, probably *walk is broken too).
testbufjoin.b is an example of how the dev's can be used.  it creates a
new fd that has a buffer at the front (e.g. leftovers from http header
reading), then continues on stdin (where the leftover may have come from).
then it reads the new fd and writes its data to stdout.

these devices are not for performance.  perhaps they make working with
one of the most basic OS concepts (fd's) a bit easier.  but perhaps this
problem is not common enough, or can be handled (with fd's preferrably)
in a better way.

mjl

[-- Attachment #2: devbuf.c --]
[-- Type: text/x-csrc, Size: 1729 bytes --]

#include	"dat.h"
#include	"fns.h"
#include	"error.h"

typedef struct Buffile Buffile;
struct Buffile
{
	uchar	*p;
	int	s;
	int	e;
};

enum
{
	Qdir,
	Qbuffile,
};

Dirtab bufdir[] =
{
	".",		{Qdir,0,QTDIR},	0,		DMDIR|0500,
	"new",		{Qbuffile},	0,		0660,
};

static Buffile*
buffilealloc(uchar *p, int n)
{
	Buffile *b;

	b = malloc(sizeof b[0]+n);
	b->p = (uchar*)b+sizeof b[0];
	memmove(b->p, p, n);
	b->s = 0;
	b->e = n;
	return b;
}

static Chan*
bufattach(char *spec)
{
	return devattach('β', spec);
}

static Walkqid*
bufwalk(Chan *c, Chan *nc, char **name, int nname)
{
	return devwalk(c, nc, name, nname, bufdir, nelem(bufdir), devgen);
}

static int
bufstat(Chan *c, uchar *db, int n)
{
        return devstat(c, db, n, bufdir, nelem(bufdir), devgen);
}

static Chan*
bufopen(Chan *c, int omode)
{
	return devopen(c, omode, bufdir, nelem(bufdir), devgen);
}

static void
bufclose(Chan *c)
{
	free(c->aux);
	c->aux = nil;
}

static long
bufread(Chan *c, void *va, long n, vlong off)
{
	Buffile *b;
	int have;

	if(c->qid.type == QTDIR)
		return devdirread(c, va, n, bufdir, nelem(bufdir), devgen);

	b = c->aux;
	if(b == nil)
		return 0;
	
	USED(off);
	have = b->e - b->s;
	if(have < n || n < 0)
		n = have;
	memmove(va, b->p, n);
	b->s += n;
	return n;
}

static long
bufwrite(Chan *c, void *va, long n, vlong off)
{
	if(c->qid.type == QTDIR)
		error(Eisdir);

	free(c->aux);
	c->aux = buffilealloc(va, n);
	return n;
}


Dev bufdevtab = {
	'β',
	"buf",

	devinit,
	bufattach,
	bufwalk,
	bufstat,
	bufopen,
	devcreate,
	bufclose,
	bufread,
	devbread,
	bufwrite,
	devbwrite,
	devremove,
	devwstat,
};

[-- Attachment #3: devjoin.c --]
[-- Type: text/x-csrc, Size: 2037 bytes --]

#include	"dat.h"
#include	"fns.h"
#include	"error.h"

typedef struct Join Join;
struct Join
{
	Chan *c;
	Join *next;
};

enum
{
	Qdir,
	Qjoinfile,
};

Dirtab joindir[] =
{
	".",		{Qdir,0,QTDIR},	0,		DMDIR|0500,
	"new",		{Qjoinfile},	0,		0660,
};

static void
joinfree(Join *j)
{
	if(j == nil)
		return;
	joinfree(j->next);
	cclose(j->c);
	free(j);
}

static Chan*
joinattach(char *spec)
{
	return devattach('δ', spec);
}

static Walkqid*
joinwalk(Chan *c, Chan *nc, char **name, int nname)
{
	return devwalk(c, nc, name, nname, joindir, nelem(joindir), devgen);
}

static int
joinstat(Chan *c, uchar *db, int n)
{
        return devstat(c, db, n, joindir, nelem(joindir), devgen);
}

static Chan*
joinopen(Chan *c, int omode)
{
	return devopen(c, omode, joindir, nelem(joindir), devgen);
}

static void
joinclose(Chan *c)
{
	joinfree(c->aux);
	c->aux = nil;
}

static long
joinread(Chan *c, void *va, long n, vlong off)
{
	Join *j;
	long l;


	if(c->qid.type == QTDIR)
		return devdirread(c, va, n, joindir, nelem(joindir), devgen);

	l = 0;
	while(c->aux != nil) {
		j = c->aux;
		l = devtab[j->c->type]->read(j->c, va, n, off);
		if(l != 0)
			break;

		c->aux = j->next;
		cclose(j->c);
		free(j);
	}
	return l;
}

static long
joinwrite(Chan *c, void *va, long n, vlong off)
{
	char buf[32];
	int fd;
	Chan *jc;
	Join *j;
	Join *nj;

	if(c->qid.type == QTDIR)
		error(Eisdir);

	if(n >= sizeof buf+1)
		error(Ebadarg);
	memmove(buf, va, n);
	buf[n] = 0;
	fd = atoi(buf);
	jc = fdtochan(up->env->fgrp, fd, -1, 0, 1);
	nj = malloc(sizeof nj[0]);
	nj->c = jc;
	nj->next = nil;
	if(c->aux == nil) {
		c->aux = nj;
	} else {
		for(j = c->aux; j->next != nil; j = j->next)
			{}
		j->next = nj;
	}
	return n;
}


Dev joindevtab = {
	'δ',
	"join",

	devinit,
	joinattach,
	joinwalk,
	joinstat,
	joinopen,
	devcreate,
	joinclose,
	joinread,
	devbread,
	joinwrite,
	devbwrite,
	devremove,
	devwstat,
};

[-- Attachment #4: testbufjoin.b --]
[-- Type: chemical/x-molconn-Z, Size: 1303 bytes --]