Subject: v15i084: Fast, multi-process dd(1) clone Newsgroups: comp.sources.unix Sender: sources Approved: rsalz@uunet.UU.NET Submitted-by: Tapani Lindgren Posting-number: Volume 15, Issue 84 Archive-name: ddd Hi! I wrote a little utility to speed up dumping to tape. It is a subset of dd(1), but has a much better throughput. I call it "ddd" for "Douple-speed DD". I posted it to eunet.sources about a month ago; a bug and some portability problems were found, but they are corrected in this second version. I planned to add some features and port the thing to Minix, but I won't have time bofere October, so it seems I've better post it now, as it is, and let others play with it too. See man page for details. Tapani Lindgren | Email or Helsinki University of Technology | or Laboratory of Information Processing Science | ------- CUT HERE ------- OUCH! ------- #! /bin/sh # This is a shell archive. Remove anything before this line, then unpack # it by saving it into a file and typing "sh file". To overwrite existing # files, type "sh file -c". You can also feed this as standard input via # unshar, or by typing "sh README <<'END_OF_README' XDDD DOUBLE-SPEED DATA DUMPER X XDdd is a stripped-down enhanced-throughput multi-process dd(1) clone. X XThis is version 2. It has some bugs fixed. The code is also cleaner Xand more portable. Forget about version 1 and destroy all copies of it. X(version 1 is the set of files _without_ version numbers...) X XSee man page and source for details. X XFUTURE PROJECETS: XImplement skip, files, count and seek opitions, Xas well as noerror and swab conversions. XSupport for multivolume files - replace bundle(1). END_OF_README if test 506 -ne `wc -c ddd.1 <<'END_OF_ddd.1' X.TH DDD 1L X.SH NAME Xddd \- double-speed data dumper X.SH SYNOPSIS X.B ddd [option=value] ... X.SH DESCRIPTION X.IR Ddd Xworks almost the same way as dd(1), but it has a much better Xthroughput, especially when used with slow i/o-devices, such as Xtape drives. The improvement is achieved mainly by dividing Xthe copying process into two processes, one of which reads while Xthe other one writes and vice versa. Also all code conversion Xcapabilities are omitted. There is no additional overhead copying Xdata between various conversion buffers. X XDdd was inspired by the vast difference in speed between BSD4.2 and XBSD4.3 dumps - in BSD4.3 dump(8) uses alternating processes to write Xto raw magnetic tape, thus keeping the tape continuously in motion. XI wanted to get the same improvement to remote dumps, so this Xfilter was needed. Directing all physical I/O through ddd usually Xincreases the throughput of any pipeline of unix commands X(if you have enough MIPS and RAM to handle two extra processes). X.SH OPTIONS XDdd uses options if, of, ibs and obs exactly as dd(1). Option bs can Xalso be used to specify ibs and obs at once. One option differs slightly Xin meaning: cbs can be used to specify the size of the internal buffer. XInput and output processes will swap duties when cbs bytes have been Xtransferred. Default values for all sizes are 512 bytes. XAs with dd(1), letters k (kilobyte), b (block) or w (word) can be Xappended to size values. XOther options are not provided. X.SH HINTS XFor best performance, block sizes should be rather large. For magnetic Xtape, I use obs=100b and cbs=500b or so. Large block sizes (~100b) are Xalso effective for network connections. However, cbs should be small Xenough for all the data to fit in core, since page faults add Xoverhead. X.SH AUTHOR XTapani Lindgren X.br XLaboratory of Information Processing Science X.br XHelsinki University of Technology X.br XFinland X.SH SEE ALSO Xdd(1), tar(1), dump(8) X.SH BUGS XShould you find one, let me know! X.SH WARNING X(Applies to U.S. residents & citizens only) X.br XDo not use this program! Get rid of it as soon as you can! XIt will probably corrupt all your data, break down your computer Xand cause severe injury to the operators. XEven reading the source code may give you a headache. XI warned you! I will take no responsibility whatsoever! END_OF_ddd.1 if test 2332 -ne `wc -c Makefile <<'END_OF_Makefile' X# Makefile for ddd X XDEFS = -DBSD XCFLAGS = -O $(DEFS) X XCC = cc XLINT = lint XCP = cp XRM = /bin/rm -f X XSRC = ddd.c XOBJ = ddd.o XHEAD = XBIN = ddd XSHAR = ddd.shar X XBINDIR = /usr/local/bin X XMAN = ddd.1 XMANDIR = /usr/local/man X Xall: $(BIN) lint X touch all X X$(BIN): $(OBJ) $(HEAD) Makefile X $(CC) $(OBJ) -o $(BIN) X Xlint: $(SRC) $(HEAD) X $(LINT) $(DEFS) $(SRC) X touch lint X Xinstall: all X strip $(BIN) X $(CP) $(BIN) $(BINDIR) X $(CP) $(MAN) $(MANDIR) X Xclean: X -$(RM) $(BIN) $(OBJ) all lint a.out core *~ #* $(SHAR) X Xshar: lint README $(MAN) Makefile $(HEAD $(SRC) X shar README $(MAN) Makefile $(HEAD) $(SRC) > $(SHAR) END_OF_Makefile if test 605 -ne `wc -c ddd.c <<'END_OF_ddd.c' X/* X * ddd.c - double dd (version 2) X * X * Copyright 1988 Helsinki University of Technology. X * All rights reserved. X * X * Permission granted to distribute, use and modify X * this code for uncommercial purposes, provided X * that this copyright notice is not changed. X * X * Author: Tapani Lindgren (nispa@cs.hut.fi) X * X * Ddd is a dd clone that operates as two processes; X * one process reads while the other one writes and vice versa. X * This way the throughput may be up to twice as good as that of dd, X * especially with slow devices such as tape drives. X * X * ***** WARNING ***** (For U.S. residents & citizens only) X * X * Do not use this program! Get rid of it as soon as you can! X * It will probably corrupt all your data, break down your computer X * and cause severe injury to the operators. X * Even reading the source code may give you a headache. X * I warned you! I will take no responsibility whatsoever! X */ X X/* declarations common to all unix versions */ X#include /* for fprintf() and stderr() */ X#include /* for SIGTERM */ Xextern char *malloc(); X X/* version dependent declarations */ X X#ifdef BSD X#include /* for union wait */ X#include /* for O_RDONLY and O_WRONLY */ Xextern char *sprintf(); X#endif X X#ifdef SYSV X#include /* for O_RDONLY and O_WRONLY */ Xvoid exit(); Xvoid perror(); X#endif X X X X/* macros to find min or max of two values */ X#define min(a,b) ((a)<(b)? (a): (b)) X#define max(a,b) ((a)>(b)? (a): (b)) X X/* inherited file descriptors */ X#define STDIN 0 X#define STDOUT 1 X X/* boolean values */ X#define FALSE 0 X#define TRUE 1 X X/* pipes have a read end and a write end */ X#define P_REND 0 X#define P_WEND 1 X X/* there are two pipes; one for read tokens and one for write tokens */ X#define RTOK_P 0 X#define WTOK_P 1 X X/* token bytes passed along pipes */ X#define TOK_CONT 0 /* go ahead */ X#define TOK_DONE 1 /* end of data */ X#define TOK_ERROR 2 /* something's wrong, you've better stop now */ X X/* input/output full/short record counters are in a table; X indexes defined below */ X#define FULLIN 0 X#define SHORTIN 1 X#define FULLOUT 2 X#define SHORTOUT 3 X X/* defaults */ X#define DEFBS 512 /* default block size */ X X/* forward declarations */ Xint doerror(); X X/* global variables */ Xstatic int X ifd, ofd, /* input/output file descriptors */ X ibs, obs, /* input/output block sizes */ X cbs, /* "conversion" buffer size */ X pid, /* pid of child (in parent) or 0 (in child) */ X eof = FALSE, /* have we encountered end-of-file */ X pipefd[2][2], /* read/write fd's for 2 pipes */ X counters[4] = {0, 0, 0, 0}, /* input/output full/short record counters */ X buflen; /* count of characters read into buffer */ Xstatic char X *buffer; /* address of buffer */ X X Xmain(argc, argv) Xint argc; Xchar *argv[]; X{ X (void) catchsignals(); /* prepare for interrupts etc */ X (void) setdefaults(); /* set default values for parameters */ X (void) parsearguments(argc, argv); /* parse arguments */ X (void) initbuffer(); /* initialize buffer */ X (void) inittokens(); /* create one READ and one WRITE token */ X (void) dofork(); /* 1 will be 2 */ X while (!eof) { /* enter main loop */ X (void) gettoken(RTOK_P); /* compete for first/next read turn */ X (void) readbuffer(); /* fill buffer with input */ X (void) gettoken(WTOK_P); /* make sure we also get the next write turn */ X /* now others may read if they wish (and if there's any data left */ X (void) passtoken(RTOK_P, eof? TOK_DONE: TOK_CONT); X (void) writebuffer(); /* ... while we write to output */ X /* this cycle is done now */ X if (!eof) (void) passtoken(WTOK_P, TOK_CONT); X } /* end of main loop */ X (void) passcounters(RTOK_P); /* send record counters to our partner */ X (void) terminate(0); /* and exit (no errors) */ X /* NOTREACHED */ X} /* end of main() */ X X Xcatchsignals() X/* arrange for some signals to be catched, so that statistics can be printed */ X{ X static int siglist[] = { X SIGINT, SIGQUIT, SIGILL, SIGFPE, X SIGBUS, SIGSEGV, SIGSYS, SIGPIPE, X SIGALRM, SIGTERM, 0 X }; X int *sigp; X X for (sigp = siglist; *sigp != 0; sigp++) X (void) signal(*sigp, doerror); X} /* end of catchsignals() */ X Xdoerror() X/* what we do if we get an error or catch a signal */ X{ X /* send error token to both pipes */ X (void) passtoken(RTOK_P, TOK_ERROR); X (void) passtoken(WTOK_P, TOK_ERROR); X /* also send i/o record counters */ X (void) passcounters(RTOK_P); X (void) passcounters(RTOK_P); X /* terminate with error status */ X (void) terminate(1); X} X Xterminate(stat) Xint stat; X{ X /* parent will try to wait for child */ X#ifdef BSD X if (pid) (void) wait((union wait *) 0); X#endif X#ifdef SYSV X if (pid) (void) wait((int *) 0); X#endif X X exit(stat); X} X Xsetdefaults() X/* set default values */ X{ X ifd = STDIN; X ofd = STDOUT; X ibs = obs = DEFBS; /* block sizes */ X cbs = 0; /* initially; will be set to max(ibs, obs, cbs) */ X} X Xparsearguments(argc, argv) Xint argc; Xchar *argv[]; X /* parse arguments */ X{ X /* constant strings "array" for recognizing options */ X static struct { X char *IF, *OF, *CBS, *IBS, *OBS, *BS, *NOTHING; X } consts = { X "if=", "of=", "cbs=", "ibs=", "obs=", "bs=", "" X }; X char **constp; /* const structure pointer */ X X for (argc--, argv++; argc > 0; argc--, argv++) { X constp = (char **) &consts; X while (**constp && strncmp(*argv, *constp, strlen(*constp))) X constp++; X /* constp now points to one of the pointers in consts structure */ X *argv += strlen(*constp); /* skip the constant part of the argument */ X if (constp == &consts.IF) { /* open another file for input */ X ifd = open(*argv, O_RDONLY); X if (ifd < 0) perror (*argv); X } else if (constp == &consts.OF) { X ofd = open(*argv, O_WRONLY | O_CREAT); /* open file for output */ X if (ofd < 0) perror (*argv); X } else if (constp == &consts.CBS) { /* set buffer size */ X cbs = evalopt(*argv); X } else if (constp == &consts.IBS) { /* set input block size */ X ibs = evalopt(*argv); X } else if (constp == &consts.OBS) { /* set output block size */ X obs = evalopt(*argv); X } else if (constp == &consts.BS) { /* set input and output block sizes */ X ibs = obs = evalopt(*argv); X } else { X (void) fprintf(stderr, X "usage: ddd [if=name] [of=name] [bs=n] [ibs=n obs=n]\n"); X exit(1); X } X } /* end of for loop */ X} /* end of parsearguments() */ X Xevalopt(p) /* return numerical value of string */ Xchar *p; X{ X int temp = 0; X X for ( ; *p >= '0' && *p <= '9'; p++) X temp = temp * 10 + *p - '0'; X if (temp < 1) { X (void) fprintf(stderr, "ddd: illegal size option\n"); X exit(1); X } X switch (*p) { X case '\0': X return(temp); X case 'w': X case 'W': X return(temp << 2); /* 4-byte words */ X case 'b': X case 'B': X return(temp << 9); /* 512-byte blocks */ X case 'k': X case 'K': X return(temp << 10); /* kilobytes */ X default: X (void) fprintf(stderr, "ddd: bad size option\n"); X exit(1); X } X /* NOTREACHED */ X} /* end of evalopt() */ X Xinitbuffer() X/* initialize buffer */ X{ X cbs = max(cbs, max(ibs, obs)); /* determine buffer size */ X if (cbs % ibs || cbs % obs) { X (void) fprintf(stderr, "ddd: warning: incompatible block/buffer sizes\n"); X } X buffer = malloc((unsigned) cbs); X if (buffer == NULL) { X (void) perror("ddd: cannot allocate buffer"); X exit(1); X } X} /* end of initbuffer() */ X Xinittokens() X/* initialize token passing system with 2 pipes */ X{ X if(pipe(pipefd[RTOK_P]) < 0 || pipe(pipefd[WTOK_P]) < 0) { X (void) perror("ddd: cannot create token pipes"); X exit(1); X } X /* create initial tokens */ X (void) passtoken(RTOK_P, TOK_CONT); X (void) passtoken(WTOK_P, TOK_CONT); X} /* end of inittokens() */ X Xpasstoken(pipenum, token) Xint pipenum; Xchar token; X/* pass a token to a pipe */ X{ X if (write(pipefd[pipenum][P_WEND], &token, 1) < 1) { X (void) perror("ddd: cannot write token to pipe"); X exit(1); X } X} /* end of passtoken() */ X Xgettoken(pipenum) Xint pipenum; X/* wait to read a token from the pipe; also see if we should stop */ X{ X char tokenbuf; X X if (read(pipefd[pipenum][P_REND], &tokenbuf, 1) < 1) { X (void) perror("ddd: cannot read token from pipe"); X exit(1); X } X if (tokenbuf != TOK_CONT) { /* we did not get what we wanted */ X (void) getcounters(pipenum); /* report record counters */ X terminate(tokenbuf == TOK_DONE); /* TOK_DONE means no error */ X } X} /* end of gettoken() */ X Xpasscounters(pipenum) Xint pipenum; X/* pass read/write counters to the other process */ X{ X if (write(pipefd[pipenum][P_WEND], (char *) counters, X sizeof(counters)) < sizeof(counters)) { X (void) perror("ddd: cannot write counters to pipe"); X exit(1); X } X} X Xgetcounters(pipenum) Xint pipenum; X/* report input/output record counts */ X{ X int hiscounters[4]; X X if (read(pipefd[pipenum][P_REND], (char *) hiscounters, X sizeof(hiscounters)) < sizeof(hiscounters)) { X (void) perror("ddd: cannot read counters from pipe"); X exit(1); X } X (void) fprintf(stderr, X "%d+%d records in\n%d+%d records out\n", X counters[FULLIN] + hiscounters[FULLIN], X counters[SHORTIN] + hiscounters[SHORTIN], X counters[FULLOUT] + hiscounters[FULLOUT], X counters[SHORTOUT] + hiscounters[SHORTOUT] X ); X} /* end of printcounters() */ X Xdofork() X/* fork into 2 processes */ X{ X if ((pid = fork()) < 0) { X (void) perror("ddd: warning: cannot fork"); X /* But continue and do our job anyway, as regular dd */ X } X} X Xreadbuffer() X/* read buffer from input */ X{ X int iolen, ioresult; X X buflen = 0; X while (buflen < cbs && !eof) { X iolen = min(ibs, cbs - buflen); X#ifdef BSD X ioresult = read(ifd, &buffer[buflen], iolen); X#endif X#ifdef SYSV X ioresult = read(ifd, &buffer[buflen], (unsigned) iolen); X#endif X if (ioresult == 0) { /* end of file */ X eof = TRUE; X } else if (ioresult < 0) { X (void) perror("ddd: read error"); X (void) doerror(); X } X buflen += ioresult; /* update current count of chars in buffer */ X /* if we got any data, update appropriate input record count */ X if (ioresult > 0) counters[(ioresult == ibs)? FULLIN: SHORTIN]++; X } X} /* end of readbuffer() */ X Xwritebuffer() X/* writing phase */ X{ X int ocount, iolen, ioresult; X X ocount = 0; /* count of chars written */ X while (ocount < buflen) { X iolen = min(obs, buflen - ocount); X#ifdef BSD X ioresult = write(ofd, &buffer[ocount], iolen); X#endif X#ifdef SYSV X ioresult = write(ofd, &buffer[ocount], (unsigned) iolen); X#endif X if (ioresult < iolen) { X perror("ddd: write error"); X (void) doerror(); X } X ocount += ioresult; X /* count output records */ X counters[(ioresult == obs)? FULLOUT: SHORTOUT]++; X } X} /* end of writebuffer() */ END_OF_ddd.c if test 10706 -ne `wc -c