Add sf(1).

Makefile

@@ -21,6 +21,7 @@ kblayout-compiler \
 mbr \
 mkinitrd \
 regress \
+sf \
 sh \
 tix \
 trianglix \
@@ -112,6 +113,7 @@ clean-build-tools:
 	$(MAKE) -C carray clean
 	$(MAKE) -C kblayout-compiler clean
 	$(MAKE) -C mkinitrd clean
+	$(MAKE) -C sf clean
 	$(MAKE) -C tix clean
 
 .PHONY: build-tools
@@ -119,6 +121,7 @@ build-tools:
 	$(MAKE) -C carray
 	$(MAKE) -C kblayout-compiler
 	$(MAKE) -C mkinitrd
+	$(MAKE) -C sf
 	$(MAKE) -C tix
 
 .PHONY: install-build-tools
@@ -126,6 +129,7 @@ install-build-tools:
 	$(MAKE) -C carray install
 	$(MAKE) -C kblayout-compiler install
 	$(MAKE) -C mkinitrd install
+	$(MAKE) -C sf install
 	$(MAKE) -C tix install
 
 .PHONY: sysroot-fsh

sf/.gitignore

@@ -0,0 +1 @@
+sf

sf/Makefile

@@ -0,0 +1,32 @@
+include ../build-aux/platform.mak
+include ../build-aux/compiler.mak
+include ../build-aux/version.mak
+include ../build-aux/dirs.mak
+
+OPTLEVEL?=$(DEFAULT_OPTLEVEL)
+CFLAGS?=$(OPTLEVEL)
+
+CFLAGS:=$(CFLAGS) -Wall -Wextra
+CPPFLAGS:=$(CPPFLAGS) -DVERSIONSTR=\"$(VERSION)\"
+
+BINARY:=sf
+
+all: $(BINARY)
+
+.PHONY: all install clean
+
+install: all
+	mkdir -p $(DESTDIR)$(BINDIR)
+	install $(BINARY) $(DESTDIR)$(BINDIR)
+	install sfnc $(DESTDIR)$(BINDIR)
+	install sfncd $(DESTDIR)$(BINDIR)
+	mkdir -p $(DESTDIR)$(MANDIR)/man1
+	install sf.1 $(DESTDIR)$(MANDIR)/man1/sf.1
+	install sfnc.1 $(DESTDIR)$(MANDIR)/man1/sfnc.1
+	install sfncd.1 $(DESTDIR)$(MANDIR)/man1/sfncd.1
+
+$(BINARY): $(BINARY).c
+	$(CC) $(CFLAGS) $(CPPFLAGS) -std=gnu11 $< -o $@
+
+clean:
+	rm -f $(BINARY) *.o

sf/sf.1

@@ -0,0 +1,64 @@
+.Dd $Mdocdate: January 7 2015 $
+.Dt SF 1
+.Os
+.Sh NAME
+.Nm sf
+.Nd serial framing
+.Sh SYNOPSIS
+.Nm sf
+.Op Fl i "|" Fl o
+.Op Ar device
+.Sh DESCRIPTION
+.Nm
+provides a simple scheme for framing a message over a byte stream.  This is
+useful in cases such as sockets, pipe, and serial devices where a real
+end of file condition would require terminating the link, but it is important
+to transmit multiple messages and keeping the link open for an arbitrary amount
+of time.
+.Pp
+.Nm
+uses a simple framing scheme with a start of message byte sequence
+.Li ( 0xF7 0xFF ) ,
+most bytes represent themselves, an escape byte sequence
+.Li ( 0xF7 0xFD ) ,
+and an end of messsage byte sequence
+.Li ( 0xF7 0xFE ) .
+UTF-8 encoded text will never need to be escaped.  Data can be recursively
+framed.
+.Pp
+Exactly one of the
+.Fl i
+and
+.Fl o
+options must be set to control whether the program is in input or output mode.
+.Pp
+Input mode works by reading one byte at a time from stdin (or the
+.Ar device
+if given).  It discards all read bytes until it finds a valid start of message
+byte sequence.  It then decodes the body and writes the decoded bytes to stdout.
+Finally it finds an end of message byte sequence and exits.
+.Pp
+Output mode works by reading bytes from stdin until an end of file condition.
+It emits a start of message byte sequence to stdout (or to the
+.Ar device
+if given).  It emits an encoded body with the contents of stdin.  Finally it
+emits an end of message byte sequence.
+.Pp
+The
+.Ar device
+argument can be a device or the path of an existing unix socket.
+.Pp
+The options are as follows:
+.Bl -tag -width "12345678"
+.It Fl i
+Decode payload.
+.It Fl o
+Encode payload.
+.El
+.Sh EXIT STATUS
+.Nm
+will exit 0 on success and non-zero otherwise.
+.Sh SEE ALSO
+.Xr sfnc 1 ,
+.Xr sfncd 1 ,
+.Xr serial-transfer 7

sf/sf.c

@@ -0,0 +1,228 @@
+/*******************************************************************************
+
+    Copyright(C) Jonas 'Sortie' Termansen 2015.
+
+    This program is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the Free
+    Software Foundation, either version 3 of the License, or (at your option)
+    any later version.
+
+    This program is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+    more details.
+
+    You should have received a copy of the GNU General Public License along with
+    this program. If not, see <http://www.gnu.org/licenses/>.
+
+    sf.c
+    Transmit and receive frames over serial connections.
+
+*******************************************************************************/
+
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/un.h>
+
+#include <err.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <locale.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+static bool try_read(int fd, const char* fd_path, unsigned char* c)
+{
+	ssize_t amount = read(fd, c, 1);
+	if ( amount < 0 )
+		err(1, "read: %s", fd_path);
+	return amount == 1;
+}
+
+static void try_write(int fd, const char* fd_path, unsigned char c)
+{
+	ssize_t amount = write(fd, &c, 1);
+	if ( amount < 0 )
+		err(1, "write: %s", fd_path);
+	if ( amount == 0 )
+		errx(1, "write: %s: End of file condition", fd_path);
+}
+
+static void receive(int fd, const char* fd_path)
+{
+	unsigned char c;
+	while ( true )
+	{
+		if ( !try_read(fd, fd_path, &c) )
+			return;
+		if ( c != 0xF7 )
+			continue;
+		if ( !try_read(fd, fd_path, &c) )
+			return;
+		if ( c == 0xFF )
+			break;
+	}
+	while ( true )
+	{
+		if ( !try_read(fd, fd_path, &c) )
+			return;
+		if ( c == 0xF7 )
+		{
+			if ( !try_read(fd, fd_path, &c) )
+				return;
+			if ( c == 0xFE )
+				break;
+			if ( c == 0xFD )
+				try_write(1, "stdout", 0xF7);
+			continue;
+		}
+		try_write(1, "stdout", c);
+	}
+}
+
+static void transmit(int fd, const char* fd_path)
+{
+	try_write(fd, fd_path, 0xF7);
+	try_write(fd, fd_path, 0xFF);
+	unsigned char c;
+	while ( try_read(0, "stdin", &c) )
+	{
+		if ( c == 0xF7 )
+		{
+			try_write(fd, fd_path, 0xF7);
+			try_write(fd, fd_path, 0xFD);
+			continue;
+		}
+		try_write(fd, fd_path, c);
+	}
+	try_write(fd, fd_path, 0xF7);
+	try_write(fd, fd_path, 0xFE);
+}
+
+static void compact_arguments(int* argc, char*** argv)
+{
+	for ( int i = 0; i < *argc; i++ )
+	{
+		while ( i < *argc && !(*argv)[i] )
+		{
+			for ( int n = i; n < *argc; n++ )
+				(*argv)[n] = (*argv)[n+1];
+			(*argc)--;
+		}
+	}
+}
+
+static void help(FILE* fp, const char* argv0)
+{
+	fprintf(fp, "Usage: %s [OPTION...] [DEVICE]\n", argv0);
+	fprintf(fp, "Transmit and receive frames over serial connections.\n");
+	fprintf(fp, "\n");
+	fprintf(fp, "Mandatory arguments to long options are mandatory for short options too.\n");
+	fprintf(fp, "  -i, --input               receive a frame\n");
+	fprintf(fp, "  -t, --output              transmit a frame\n");
+	fprintf(fp, "      --help                display this help and exit\n");
+	fprintf(fp, "      --version             output version information and exit\n");
+}
+
+static void version(FILE* fp, const char* argv0)
+{
+	fprintf(fp, "%s (Sortix) %s\n", argv0, VERSIONSTR);
+	fprintf(fp, "License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>.\n");
+	fprintf(fp, "This is free software: you are free to change and redistribute it.\n");
+	fprintf(fp, "There is NO WARRANTY, to the extent permitted by law.\n");
+}
+
+int main(int argc, char* argv[])
+{
+	setlocale(LC_ALL, "");
+
+	bool flag_input = false;
+	bool flag_output = false;
+
+	const char* argv0 = argv[0];
+	for ( int i = 1; i < argc; i++ )
+	{
+		const char* arg = argv[i];
+		if ( arg[0] != '-' || !arg[1] )
+			continue;
+		argv[i] = NULL;
+		if ( !strcmp(arg, "--") )
+			break;
+		if ( arg[1] != '-' )
+		{
+			char c;
+			while ( (c = *++arg) ) switch ( c )
+			{
+			case 'i': flag_input = true; break;
+			case 'o': flag_output = true; break;
+			default:
+				fprintf(stderr, "%s: unknown option -- '%c'\n", argv0, c);
+				help(stderr, argv0);
+				exit(1);
+			}
+		}
+		else if ( !strcmp(arg, "--help") )
+			help(stdout, argv0), exit(0);
+		else if ( !strcmp(arg, "--version") )
+			version(stdout, argv0), exit(0);
+		else if ( !strcmp(arg, "--input") )
+			flag_input = true;
+		else if ( !strcmp(arg, "--output") )
+			flag_output = true;
+		else
+		{
+			fprintf(stderr, "%s: unknown option: %s\n", argv0, arg);
+			help(stderr, argv0);
+			exit(1);
+		}
+	}
+
+	compact_arguments(&argc, &argv);
+
+	if ( flag_input + flag_output == 0 )
+		errx(1, "need to specify exactly one of the -io options");
+
+	if ( flag_input + flag_output > 1 )
+		errx(1, "specified multiple of the incompatible -io options");
+
+	if ( 3 <= argc )
+		errx(1, "extra operand `%s'", argv[2]);
+
+	int fd = flag_input ? 0 : 1;
+	const char* fd_path = flag_input ? "stdin" : "stdout";
+
+	if ( 2 <= argc )
+	{
+		fd_path = argv[1];
+		struct stat st;
+		if ( stat(fd_path, &st) == 0 && S_ISSOCK(st.st_mode) )
+		{
+			if ( (fd = socket(AF_UNIX, SOCK_STREAM, 0)) < 0 )
+				err(1, "socket");
+			struct sockaddr_un* addr;
+			size_t addr_size = sizeof(*addr) - sizeof(addr->sun_path) +
+			                   strlen(fd_path) + 1;
+			if ( addr_size < sizeof(*addr) )
+				addr_size = sizeof(*addr);
+			if ( !(addr = (struct sockaddr_un*) malloc(addr_size)) )
+				err(1, "malloc");
+			memset(addr, 0, addr_size);
+			addr->sun_family = AF_UNIX;
+			strcpy(addr->sun_path, fd_path);
+			if ( connect(fd, (const struct sockaddr*) addr, addr_size) < 0 )
+				err(1, "connect: %s", fd_path);
+		}
+		else if ( (fd = open(fd_path, flag_input ? O_RDONLY : O_WRONLY)) < 0 )
+			err(1, "%s", fd_path);
+	}
+
+	(flag_input ? receive : transmit)(fd, fd_path);
+
+	if ( fd != 0 && fd != 1 )
+		close(fd);
+
+	return 0;
+}

sf/sfnc

@@ -0,0 +1,30 @@
+#!/bin/sh -e
+################################################################################
+#
+#   Copyright(C) Jonas 'Sortie' Termansen 2016.
+#
+#   This program is free software: you can redistribute it and/or modify it
+#   under the terms of the GNU General Public License as published by the Free
+#   Software Foundation, either version 3 of the License, or (at your option)
+#   any later version.
+#
+#   This program is distributed in the hope that it will be useful, but WITHOUT
+#   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+#   more details.
+#
+#   You should have received a copy of the GNU General Public License along with
+#   this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#   sfnc
+#   nc over sf
+#
+################################################################################
+
+if [ $# != 4 ]; then
+  echo Usage: $0 host port transmit-device receive-device
+  exit 1
+fi
+(printf -- "$1\n" | sf -o &&
+ printf -- "$2\n" | sf -o &&
+ cat) | sf -o -- "$3" | sf -i -- "$4"

sf/sfnc.1

@@ -0,0 +1,42 @@
+.Dd $Mdocdate: February 4 2015 $
+.Dt SFNC 1
+.Os
+.Sh NAME
+.Nm sfnc
+.Nd network connection over sf (client side)
+.Sh SYNOPSIS
+.Nm sfnc
+.Ar host
+.Ar port
+.Ar transmit-device
+.Ar receive-device
+.Sh DESCRIPTION
+.Nm
+communicates with another program over two byte steams and asks the remote
+program (usually
+.Xr sfncd 1 )
+to connect to the specified
+.Ar host
+and
+.Ar port .
+.Pp
+It reads bytes from the standard input and transmits them across
+.Ar transmit-device .
+It receives bytes from
+.Ar receive-device
+and sends them to the standard output.
+.Pp
+.Ss Protocol
+The client transmits in one
+.Xr sf 1
+session, which contains two nested
+.Xr sf 1
+connections (first host, then port), and then the actual body read from the
+standard input.  The server transmits in one
+.Xr sf 1
+session, which begins once the host and port has been received, and it contains
+the response.
+.Sh SEE ALSO
+.Xr sf 1 ,
+.Xr sfncd 1 ,
+.Xr serial-transfer 7

sf/sfncd

@@ -0,0 +1,35 @@
+#!/bin/sh -e
+################################################################################
+#
+#   Copyright(C) Jonas 'Sortie' Termansen 2016.
+#
+#   This program is free software: you can redistribute it and/or modify it
+#   under the terms of the GNU General Public License as published by the Free
+#   Software Foundation, either version 3 of the License, or (at your option)
+#   any later version.
+#
+#   This program is distributed in the hope that it will be useful, but WITHOUT
+#   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#   FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+#   more details.
+#
+#   You should have received a copy of the GNU General Public License along with
+#   this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#   sfncd
+#   Remote for nc over sf
+#
+################################################################################
+
+if [ $# -lt 2 ] || [ 3 -lt $# ]; then
+  echo Usage: $0 receive-device transmit-device [server-command]
+  exit 1
+fi
+sf -i -- "$1" |
+(export HOST=$(sf -i) &&
+ export PORT=$(sf -i) &&
+ (if [ $# -lt 3 ]; then
+    nc -- "$HOST" "$PORT"
+  else
+    sh -c "$3"
+  fi) | sf -o -- "$2")

sf/sfncd.1

@@ -0,0 +1,36 @@
+.Dd $Mdocdate: February 4 2015 $
+.Dt SFNCD 1
+.Os
+.Sh NAME
+.Nm sfncd
+.Nd network connection over sf (server side)
+.Sh SYNOPSIS
+.Nm sfncd
+.Ar transmit-device
+.Ar receive-device
+.Op Ar server-command
+.Sh DESCRIPTION
+.Nm
+communicates with another program over two byte steams and receives a host and
+port which it connects to using
+.Xr nc 1
+or
+.Ar server-command
+if specified (with the
+.Ev HOST
+and
+.Ev PORT
+environment variables set to the parameters).
+.Pp
+It receives bytes from
+.Ar receive-device
+and sends them to the invoked program.  It reads bytes from the invoked program
+and transmits them through
+.Ar transmit-device .
+.Pp
+The protocol is as described in
+.Xr sfnc 1 .
+.Sh SEE ALSO
+.Xr sf 1 ,
+.Xr sfnc 1 ,
+.Xr serial-transfer 7

share/man/man7/development.7

@@ -194,6 +194,8 @@ kblayout-compiler
 .It
 mkinitrd
 .It
+sf
+.It
 tix
 .El
 .Pp

share/man/man7/serial-transfer.7

@@ -0,0 +1,130 @@
+.Dd $Mdocdate: January 6 2016 $
+.Dt SERIAL-TRANSFER 7
+.Os
+.Sh NAME
+.Nm serial-transfer
+.Nd files over serial device
+.Sh DESCRIPTION
+You can transfer data using the serial line.  The
+.Xr sf 1
+program provides simple framing which is useful to conduct advanced transfers
+by combining with other tools such as
+.Xr tar 1 .
+.Pp
+The serial device will be available at the appropriate device after boot, such
+as
+.Pa /dev/com1 .
+Data written to it will be available to readers on the other end and likewise
+data written on the other end will be available to local readers.  Take care to
+ensure that the reader is always reading before writing or you may lose data.
+.Pp
+You get a byte stream between the guest and host using this interface. This is
+powerful but often you want to transfer finite payloads and have the transfer
+finish when done rather than needing to manually interrupt it.
+.Pp
+The
+.Xr sf 1
+program encodes and decodes frames.
+.Li sf -o
+will emit a start byte
+sequence, then read from stdin and encode a body, and finally emit an end
+sequence.
+.Li sf -i
+will read bytes until it finds a start sequence, then it will decode the body
+and emit it to stdout, and finally stop when it receives the end sequence. We
+can use this to do transfers over the serial connection.
+.Pp
+.Xr sf 1
+is a Sortix specific program. Other operating systems don't come with it and you
+need to build it from the Sortix source code. This is automatically done by the
+.Sy build-tools
+target during
+.Xr cross-development 7 .
+You can also just transfer its code from
+.Pa /src/sf/sf.c
+over the serial line.
+.Ss Virtual Machines
+This method is useful when running inside a virtual machine and you wish to
+communicate with the host system.  This is particularly useful if you connect
+the serial line to a unix socket. In Qemu, this is done with:
+.Bd -literal
+   -serial unix:/tmp/serial,server,nowait
+.Ed
+.Pp
+In VirtualBox, in the virtual machine settings, under serial ports, enable one
+and put it in mode Host Pipe and mark Create Pipe.
+.Ss Conventions
+Let
+.Pa /dev/receiver
+mean the device on the receiving machine and let
+.Pa /dev/transmitter
+mean the device on the transmitting machine.  This will be devices such as
+.Pa /dev/com1 .
+If one end is the host of a virtual machine as described above, its device will
+be an unix socket such as
+.Pa /tmp/serial .
+.Ss Simple File Transfer
+You can then transfer a file from this system to another. First run on the
+receiving machine:
+.Bd -literal
+    sf -i /dev/receiver > file.txt
+.Ed
+.Pp
+Then run on the transmitting machine:
+.Bd -literal
+    sf -o /dev/transmitter < file.txt
+.Ed
+.Pp
+The sender will stop when it has transmitted the last byte and the receiver will
+end when it has recognized an end sequence.
+.Ss Advanced File Transfer
+You can transfer multiple files using
+.Xr tar 1 .
+This also allows you to preserve file meta data such as permissions and modified
+time.  First run on the receiving machine:
+.Bd -literal
+    sf -i /dev/receiver | tar -xv
+.Ed
+.Pp
+Then run on the transmitting machine:
+.Bd -literal
+    tar -cv *.patch | sf -o /dev/transmitter
+.Ed
+.Pp
+The
+.Fl v
+option is useful as it displays the names of files as they are transferred.
+.Ss Network Connection
+It is possible to use the
+.Xr sfnc 1
+and
+.Xr sfncd 1
+scripts to create a bidirectional communication channel using two serial ports,
+one for each direction.  The scripts use a protocol where
+.Xr sfnc 1
+sends a hostname, a port, and the body from stdin.  Likewise the
+.Xr sfncd 1
+script receives the two parameters and invokes
+.Xr nc 1
+(or another program as specified).
+.Pp
+For instance, run on the server:
+.Bd -literal
+    sfncd /dev/receive /dev/transmit
+.Ed
+.Pp
+And then run on the client:
+.Bd -literal
+    sfnc irc.freenode.net 6667 /dev/receive /dev/transmit
+.Ed
+.Pp
+This will last for the duration of the connection.
+.Xr sfncd 1
+needs to be run again to start another connection.  This scheme only allows one
+connection at one given time, but with it is possible for custom programs on
+either side to multiplex connections.
+.Sh SEE ALSO
+.Xr sf 1 ,
+.Xr tar 1 ,
+.Xr development 7 ,
+.Xr user-guide 7

share/man/man7/user-guide.7

@@ -104,6 +104,9 @@ ext2 extensions.  You can make a compatible filesystem with:
 Sortix does not have networking at this time.  Unix sockets have a basic
 implementation incapable of advanced features.  The standard library and
 kernel provides stubs for many network interfaces.
+.Ss Serial Transfer
+It is possible to transfer files over serial devices as described in
+.Xr serial-transfer 7 .
 .Ss Development
 The system is self-hosting and is capable of building itself as described in
 .Xr development 7 .
@@ -112,4 +115,5 @@ Ports are cross-compiled as described in
 but it is becoming feasible to build a large number of them natively.
 .Sh SEE ALSO
 .Xr cross-development 7 ,
-.Xr development 7
+.Xr development 7 ,
+.Xr serial-transfer 7