From mboxrd@z Thu Jan  1 00:00:00 1970
MIME-Version: 1.0
From: Ole-Hjalmar Kristensen <ole.hjalmar.kristensen@gmail.com>
Date: Tue, 12 Dec 2017 10:33:55 +0100
Message-ID: <CAHqDL_8v9T77KZhC7XuDVQTHMBXyGDu+KJUBOjdPQ-bLb-POMQ@mail.gmail.com>
To: Fans of the OS Plan 9 from Bell Labs <9fans@9fans.net>
Content-Type: multipart/alternative; boundary="94eb2c19c6389003cf056021598b"
Subject: [9fans] A potentially useful venti client
Topicbox-Message-UUID: c5fa7a88-ead9-11e9-9d60-3106f5b1d025

--94eb2c19c6389003cf056021598b
Content-Type: text/plain; charset="UTF-8"

Based on copy.c and readlist.c, I have cobbled together a venti client to
copy a list of venti blocks from one venti server to another. I am thinking
of using it to incrementally replicate the contents on one site site to
another. It could even be used for two-way replication, since the CAS and
deduplicating properties of venti ensures that you will never have write
conflicts at a block level.

I have tried it out by feeding it with the output from printarenas, and it
seems to work reasonably well. Does anyone have any good ideas about how to
incrementally extract the set of scores that has been added to a venti
server? You could extract the whole set of scores and do a diff with an old
set of course, but that's rather inefficient.

Ole-Hj.


#include <u.h>
#include <libc.h>
#include <thread.h>
#include <venti.h>
#include <bio.h>

enum
{
    // XXX What to do here?
    VtMaxLumpSize = 65535,
};

char *srchost;
char *dsthost;
Biobuf b;
VtConn *zsrc;
VtConn *zdst;
uchar *buf;
void run(Biobuf*);
int nn;

void
usage(void)
{
    fprint(2, "usage: copylist srchost dsthost list\n");
    threadexitsall("usage");
}

int
parsescore(uchar *score, char *buf, int n)
{
    int i, c;

    memset(score, 0, VtScoreSize);

    if(n != VtScoreSize*2){
        werrstr("score wrong length %d", n);
        return -1;
    }
    for(i=0; i<VtScoreSize*2; i++) {
        if(buf[i] >= '0' && buf[i] <= '9')
            c = buf[i] - '0';
        else if(buf[i] >= 'a' && buf[i] <= 'f')
            c = buf[i] - 'a' + 10;
        else if(buf[i] >= 'A' && buf[i] <= 'F')
            c = buf[i] - 'A' + 10;
        else {
            c = buf[i];
            werrstr("bad score char %d '%c'", c, c);
            return -1;
        }

        if((i & 1) == 0)
            c <<= 4;

        score[i>>1] |= c;
    }
    return 0;
}

void
threadmain(int argc, char *argv[])
{
    int fd, i;

    ARGBEGIN{
    default:
        usage();
        break;
    }ARGEND

    if(argc < 2)
        usage();

    fmtinstall('V', vtscorefmt);
    buf = vtmallocz(VtMaxLumpSize);

    srchost = argv[0];
    zsrc = vtdial(srchost);
    if(zsrc == nil)
        sysfatal("could not dial src server: %r");
    if(vtconnect(zsrc) < 0)
        sysfatal("vtconnect src: %r");

    dsthost = argv[1];
    zdst = vtdial(dsthost);
    if(zdst == nil)
        sysfatal("could not dial dst server: %r");
    if(vtconnect(zdst) < 0)
        sysfatal("vtconnect dst: %r");

    if(argc == 2){
        Binit(&b, 0, OREAD);
        run(&b);
    }else{
        for(i=2; i<argc; i++){
            if((fd = open(argv[i], OREAD)) < 0)
                sysfatal("open %s: %r", argv[i]);
            Binit(&b, fd, OREAD);
            run(&b);
        }
    }
    threadexitsall(nil);
}

void
run(Biobuf *b)
{
    char *p, *f[10];
    int nf;
    uchar score[20];
    int type, n;

    while((p = Brdline(b, '\n')) != nil){
        p[Blinelen(b)-1] = 0;
        nf = tokenize(p, f, nelem(f));
        if(nf != 2)
            sysfatal("syntax error in work list");
        if(parsescore(score, f[0], strlen(f[0])) < 0)
            sysfatal("bad score %s in work list", f[0]);
        type = atoi(f[1]);
        n = vtread(zsrc, score, type, buf, VtMaxLumpSize);
        if(n < 0)
            sysfatal("could not read %s %s: %r", f[0], f[1]);
        n = vtwrite(zdst, score, type, buf, n);
        if(n < 0)
            sysfatal("could not write %s %s: %r", f[0], f[1]);
        if(++nn%1000 == 0)
            print("%d...", nn);
    }
}

--94eb2c19c6389003cf056021598b
Content-Type: text/html; charset="UTF-8"
Content-Transfer-Encoding: base64

PGRpdiBkaXI9Imx0ciI+QmFzZWQgb24gY29weS5jIGFuZCByZWFkbGlzdC5jLCBJIGhhdmUgY29i
YmxlZCB0b2dldGhlciBhIHZlbnRpIGNsaWVudCB0byBjb3B5IGEgbGlzdCBvZiB2ZW50aSBibG9j
a3MgZnJvbSBvbmUgdmVudGkgc2VydmVyIHRvIGFub3RoZXIuIEkgYW0gdGhpbmtpbmcgb2YgdXNp
bmcgaXQgdG8gaW5jcmVtZW50YWxseSByZXBsaWNhdGUgdGhlIGNvbnRlbnRzIG9uIG9uZSBzaXRl
IHNpdGUgdG8gYW5vdGhlci4gSXQgY291bGQgZXZlbiBiZSB1c2VkIGZvciB0d28td2F5IHJlcGxp
Y2F0aW9uLCBzaW5jZSB0aGUgQ0FTIGFuZCBkZWR1cGxpY2F0aW5nIHByb3BlcnRpZXMgb2YgdmVu
dGkgZW5zdXJlcyB0aGF0IHlvdSB3aWxsIG5ldmVyIGhhdmUgd3JpdGUgY29uZmxpY3RzIGF0IGEg
YmxvY2sgbGV2ZWwuPGJyPjxkaXY+PGRpdj48YnI+PC9kaXY+PGRpdj5JIGhhdmUgdHJpZWQgaXQg
b3V0IGJ5IGZlZWRpbmcgaXQgd2l0aCB0aGUgb3V0cHV0IGZyb20gcHJpbnRhcmVuYXMsIGFuZCBp
dCBzZWVtcyB0byB3b3JrIHJlYXNvbmFibHkgd2VsbC4gRG9lcyBhbnlvbmUgaGF2ZSBhbnkgZ29v
ZCBpZGVhcyBhYm91dCBob3cgdG8gaW5jcmVtZW50YWxseSBleHRyYWN0IHRoZSBzZXQgb2Ygc2Nv
cmVzIHRoYXQgaGFzIGJlZW4gYWRkZWQgdG8gYSB2ZW50aSBzZXJ2ZXI/IFlvdSBjb3VsZCBleHRy
YWN0IHRoZSB3aG9sZSBzZXQgb2Ygc2NvcmVzIGFuZCBkbyBhIGRpZmYgd2l0aCBhbiBvbGQgc2V0
IG9mIGNvdXJzZSwgYnV0IHRoYXQmIzM5O3MgcmF0aGVyIGluZWZmaWNpZW50LjwvZGl2PjxkaXY+
PGJyPjwvZGl2PjxkaXY+T2xlLUhqLjxicj48L2Rpdj48ZGl2Pjxicj48L2Rpdj48ZGl2Pjxicj48
L2Rpdj48ZGl2PiNpbmNsdWRlICZsdDt1LmgmZ3Q7PGJyPiNpbmNsdWRlICZsdDtsaWJjLmgmZ3Q7
PGJyPiNpbmNsdWRlICZsdDt0aHJlYWQuaCZndDs8YnI+I2luY2x1ZGUgJmx0O3ZlbnRpLmgmZ3Q7
PGJyPiNpbmNsdWRlICZsdDtiaW8uaCZndDs8YnI+PGJyPmVudW08YnI+ezxicj7CoMKgwqAgLy8g
WFhYIFdoYXQgdG8gZG8gaGVyZT88YnI+wqDCoMKgIFZ0TWF4THVtcFNpemUgPSA2NTUzNSw8YnI+
fTs8YnI+PGJyPmNoYXIgKnNyY2hvc3Q7PGJyPmNoYXIgKmRzdGhvc3Q7PGJyPkJpb2J1ZiBiOzxi
cj5WdENvbm4gKnpzcmM7PGJyPlZ0Q29ubiAqemRzdDs8YnI+dWNoYXIgKmJ1Zjs8YnI+dm9pZCBy
dW4oQmlvYnVmKik7PGJyPmludCBubjs8YnI+PGJyPnZvaWQ8YnI+dXNhZ2Uodm9pZCk8YnI+ezxi
cj7CoMKgwqAgZnByaW50KDIsICZxdW90O3VzYWdlOiBjb3B5bGlzdCBzcmNob3N0IGRzdGhvc3Qg
bGlzdFxuJnF1b3Q7KTs8YnI+wqDCoMKgIHRocmVhZGV4aXRzYWxsKCZxdW90O3VzYWdlJnF1b3Q7
KTs8YnI+fTxicj48YnI+aW50PGJyPnBhcnNlc2NvcmUodWNoYXIgKnNjb3JlLCBjaGFyICpidWYs
IGludCBuKTxicj57PGJyPsKgwqDCoCBpbnQgaSwgYzs8YnI+PGJyPsKgwqDCoCBtZW1zZXQoc2Nv
cmUsIDAsIFZ0U2NvcmVTaXplKTs8YnI+PGJyPsKgwqDCoCBpZihuICE9IFZ0U2NvcmVTaXplKjIp
ezxicj7CoMKgwqAgwqDCoMKgIHdlcnJzdHIoJnF1b3Q7c2NvcmUgd3JvbmcgbGVuZ3RoICVkJnF1
b3Q7LCBuKTs8YnI+wqDCoMKgIMKgwqDCoCByZXR1cm4gLTE7PGJyPsKgwqDCoCB9PGJyPsKgwqDC
oCBmb3IoaT0wOyBpJmx0O1Z0U2NvcmVTaXplKjI7IGkrKykgezxicj7CoMKgwqAgwqDCoMKgIGlm
KGJ1ZltpXSAmZ3Q7PSAmIzM5OzAmIzM5OyAmYW1wOyZhbXA7IGJ1ZltpXSAmbHQ7PSAmIzM5Ozkm
IzM5Oyk8YnI+wqDCoMKgIMKgwqDCoCDCoMKgwqAgYyA9IGJ1ZltpXSAtICYjMzk7MCYjMzk7Ozxi
cj7CoMKgwqAgwqDCoMKgIGVsc2UgaWYoYnVmW2ldICZndDs9ICYjMzk7YSYjMzk7ICZhbXA7JmFt
cDsgYnVmW2ldICZsdDs9ICYjMzk7ZiYjMzk7KTxicj7CoMKgwqAgwqDCoMKgIMKgwqDCoCBjID0g
YnVmW2ldIC0gJiMzOTthJiMzOTsgKyAxMDs8YnI+wqDCoMKgIMKgwqDCoCBlbHNlIGlmKGJ1Zltp
XSAmZ3Q7PSAmIzM5O0EmIzM5OyAmYW1wOyZhbXA7IGJ1ZltpXSAmbHQ7PSAmIzM5O0YmIzM5Oyk8
YnI+wqDCoMKgIMKgwqDCoCDCoMKgwqAgYyA9IGJ1ZltpXSAtICYjMzk7QSYjMzk7ICsgMTA7PGJy
PsKgwqDCoCDCoMKgwqAgZWxzZSB7PGJyPsKgwqDCoCDCoMKgwqAgwqDCoMKgIGMgPSBidWZbaV07
PGJyPsKgwqDCoCDCoMKgwqAgwqDCoMKgIHdlcnJzdHIoJnF1b3Q7YmFkIHNjb3JlIGNoYXIgJWQg
JiMzOTslYyYjMzk7JnF1b3Q7LCBjLCBjKTs8YnI+wqDCoMKgIMKgwqDCoCDCoMKgwqAgcmV0dXJu
IC0xOzxicj7CoMKgwqAgwqDCoMKgIH08YnI+PGJyPsKgwqDCoCDCoMKgwqAgaWYoKGkgJmFtcDsg
MSkgPT0gMCk8YnI+wqDCoMKgIMKgwqDCoCDCoMKgwqAgYyAmbHQ7Jmx0Oz0gNDs8YnI+wqDCoMKg
IDxicj7CoMKgwqAgwqDCoMKgIHNjb3JlW2kmZ3Q7Jmd0OzFdIHw9IGM7PGJyPsKgwqDCoCB9PGJy
PsKgwqDCoCByZXR1cm4gMDs8YnI+fTxicj48YnI+dm9pZDxicj50aHJlYWRtYWluKGludCBhcmdj
LCBjaGFyICphcmd2W10pPGJyPns8YnI+wqDCoMKgIGludCBmZCwgaTs8YnI+PGJyPsKgwqDCoCBB
UkdCRUdJTns8YnI+wqDCoMKgIGRlZmF1bHQ6PGJyPsKgwqDCoCDCoMKgwqAgdXNhZ2UoKTs8YnI+
wqDCoMKgIMKgwqDCoCBicmVhazs8YnI+wqDCoMKgIH1BUkdFTkQ8YnI+PGJyPsKgwqDCoCBpZihh
cmdjICZsdDsgMik8YnI+wqDCoMKgIMKgwqDCoCB1c2FnZSgpOzxicj48YnI+wqDCoMKgIGZtdGlu
c3RhbGwoJiMzOTtWJiMzOTssIHZ0c2NvcmVmbXQpOzxicj7CoMKgwqAgYnVmID0gdnRtYWxsb2N6
KFZ0TWF4THVtcFNpemUpOzxicj7CoMKgwqAgPGJyPsKgwqDCoCBzcmNob3N0ID0gYXJndlswXTs8
YnI+wqDCoMKgIHpzcmMgPSB2dGRpYWwoc3JjaG9zdCk7PGJyPsKgwqDCoCBpZih6c3JjID09IG5p
bCk8YnI+wqDCoMKgIMKgwqDCoCBzeXNmYXRhbCgmcXVvdDtjb3VsZCBub3QgZGlhbCBzcmMgc2Vy
dmVyOiAlciZxdW90Oyk7PGJyPsKgwqDCoCBpZih2dGNvbm5lY3QoenNyYykgJmx0OyAwKTxicj7C
oMKgwqAgwqDCoMKgIHN5c2ZhdGFsKCZxdW90O3Z0Y29ubmVjdCBzcmM6ICVyJnF1b3Q7KTs8YnI+
PGJyPsKgwqDCoCBkc3Rob3N0ID0gYXJndlsxXTs8YnI+wqDCoMKgIHpkc3QgPSB2dGRpYWwoZHN0
aG9zdCk7PGJyPsKgwqDCoCBpZih6ZHN0ID09IG5pbCk8YnI+wqDCoMKgIMKgwqDCoCBzeXNmYXRh
bCgmcXVvdDtjb3VsZCBub3QgZGlhbCBkc3Qgc2VydmVyOiAlciZxdW90Oyk7PGJyPsKgwqDCoCBp
Zih2dGNvbm5lY3QoemRzdCkgJmx0OyAwKTxicj7CoMKgwqAgwqDCoMKgIHN5c2ZhdGFsKCZxdW90
O3Z0Y29ubmVjdCBkc3Q6ICVyJnF1b3Q7KTs8YnI+PGJyPsKgwqDCoCBpZihhcmdjID09IDIpezxi
cj7CoMKgwqAgwqDCoMKgIEJpbml0KCZhbXA7YiwgMCwgT1JFQUQpOzxicj7CoMKgwqAgwqDCoMKg
IHJ1bigmYW1wO2IpOzxicj7CoMKgwqAgfWVsc2V7PGJyPsKgwqDCoCDCoMKgwqAgZm9yKGk9Mjsg
aSZsdDthcmdjOyBpKyspezxicj7CoMKgwqAgwqDCoMKgIMKgwqDCoCBpZigoZmQgPSBvcGVuKGFy
Z3ZbaV0sIE9SRUFEKSkgJmx0OyAwKTxicj7CoMKgwqAgwqDCoMKgIMKgwqDCoCDCoMKgwqAgc3lz
ZmF0YWwoJnF1b3Q7b3BlbiAlczogJXImcXVvdDssIGFyZ3ZbaV0pOzxicj7CoMKgwqAgwqDCoMKg
IMKgwqDCoCBCaW5pdCgmYW1wO2IsIGZkLCBPUkVBRCk7PGJyPsKgwqDCoCDCoMKgwqAgwqDCoMKg
IHJ1bigmYW1wO2IpOzxicj7CoMKgwqAgwqDCoMKgIH08YnI+wqDCoMKgIH08YnI+wqDCoMKgIHRo
cmVhZGV4aXRzYWxsKG5pbCk7PGJyPn08YnI+PGJyPnZvaWQ8YnI+cnVuKEJpb2J1ZiAqYik8YnI+
ezxicj7CoMKgwqAgY2hhciAqcCwgKmZbMTBdOzxicj7CoMKgwqAgaW50IG5mOzxicj7CoMKgwqAg
dWNoYXIgc2NvcmVbMjBdOzxicj7CoMKgwqAgaW50IHR5cGUsIG47PGJyPjxicj7CoMKgwqAgd2hp
bGUoKHAgPSBCcmRsaW5lKGIsICYjMzk7XG4mIzM5OykpICE9IG5pbCl7PGJyPsKgwqDCoCDCoMKg
wqAgcFtCbGluZWxlbihiKS0xXSA9IDA7PGJyPsKgwqDCoCDCoMKgwqAgbmYgPSB0b2tlbml6ZShw
LCBmLCBuZWxlbShmKSk7PGJyPsKgwqDCoCDCoMKgwqAgaWYobmYgIT0gMik8YnI+wqDCoMKgIMKg
wqDCoCDCoMKgwqAgc3lzZmF0YWwoJnF1b3Q7c3ludGF4IGVycm9yIGluIHdvcmsgbGlzdCZxdW90
Oyk7PGJyPsKgwqDCoCDCoMKgwqAgaWYocGFyc2VzY29yZShzY29yZSwgZlswXSwgc3RybGVuKGZb
MF0pKSAmbHQ7IDApPGJyPsKgwqDCoCDCoMKgwqAgwqDCoMKgIHN5c2ZhdGFsKCZxdW90O2JhZCBz
Y29yZSAlcyBpbiB3b3JrIGxpc3QmcXVvdDssIGZbMF0pOzxicj7CoMKgwqAgwqDCoMKgIHR5cGUg
PSBhdG9pKGZbMV0pOzxicj7CoMKgwqAgwqDCoMKgIG4gPSB2dHJlYWQoenNyYywgc2NvcmUsIHR5
cGUsIGJ1ZiwgVnRNYXhMdW1wU2l6ZSk7PGJyPsKgwqDCoCDCoMKgwqAgaWYobiAmbHQ7IDApPGJy
PsKgwqDCoCDCoMKgwqAgwqDCoMKgIHN5c2ZhdGFsKCZxdW90O2NvdWxkIG5vdCByZWFkICVzICVz
OiAlciZxdW90OywgZlswXSwgZlsxXSk7PGJyPsKgwqDCoCDCoMKgwqAgbiA9IHZ0d3JpdGUoemRz
dCwgc2NvcmUsIHR5cGUsIGJ1Ziwgbik7PGJyPsKgwqDCoCDCoMKgwqAgaWYobiAmbHQ7IDApPGJy
PsKgwqDCoCDCoMKgwqAgwqDCoMKgIHN5c2ZhdGFsKCZxdW90O2NvdWxkIG5vdCB3cml0ZSAlcyAl
czogJXImcXVvdDssIGZbMF0sIGZbMV0pOzxicj7CoMKgwqAgwqDCoMKgIGlmKCsrbm4lMTAwMCA9
PSAwKTxicj7CoMKgwqAgwqDCoMKgIMKgwqDCoCBwcmludCgmcXVvdDslZC4uLiZxdW90Oywgbm4p
Ozxicj7CoMKgwqAgfTxicj59PGJyPjxicj48L2Rpdj48L2Rpdj48L2Rpdj4NCg==
--94eb2c19c6389003cf056021598b--