Refactor fcntl(2) ABI.

This is an incompatible ABI change.

libc/fcntl/fcntl.cpp

@@ -24,16 +24,26 @@
 
 #include <sys/syscall.h>
 
+#include <errno.h>
 #include <fcntl.h>
 #include <stdarg.h>
+#include <stdint.h>
 
-DEFN_SYSCALL3(int, sys_fcntl, SYSCALL_FCNTL, int, int, unsigned long);
+DEFN_SYSCALL3(int, sys_fcntl, SYSCALL_FCNTL, int, int, uintptr_t);
 
 extern "C" int fcntl(int fd, int cmd, ...)
 {
+	uintptr_t arg;
 	va_list ap;
 	va_start(ap, cmd);
-	unsigned long arg = va_arg(ap, unsigned long);
+	switch ( F_DECODE_CMD_TYPE(cmd) )
+	{
+	case F_TYPE_VOID: arg = 0; break;
+	case F_TYPE_INT: arg = (uintptr_t) va_arg(ap, int); break;
+	case F_TYPE_LONG: arg = (uintptr_t) va_arg(ap, long); break;
+	case F_TYPE_PTR: arg = (uintptr_t) va_arg(ap, void*); break;
+	default: return errno = EINVAL, -1;
+	}
 	va_end(ap);
 	return sys_fcntl(fd, cmd, arg);
 }

sortix/include/sortix/fcntl.h

@@ -50,10 +50,43 @@ __BEGIN_DECLS
 
 #define __FD_ALLOWED_FLAGS (FD_CLOEXEC | FD_CLOFORK)
 
-#define F_SETFD 0
-#define F_GETFD 1
-#define F_SETFL 2
-#define F_GETFL 3
+/* Encode type information about arguments into fcntl commands. Unfortunately
+   the fcntl function declaration doesn't include type information, which means
+   that the fcntl implementation either needs a list of command type information
+   to use va_list correctly - or we can simply embed the information into the
+   commands. */
+#define F_TYPE_EXP 3 /* 2^3 kinds of argument types supported.*/
+#define F_TYPE_VOID 0
+#define F_TYPE_INT 1
+#define F_TYPE_LONG 2
+#define F_TYPE_PTR 3
+/* 5-7 is unused in case of future expansion. */
+#define F_ENCODE_CMD(cmd_val, arg_type) ((cmd_val) << F_TYPE_EXP | (arg_type))
+#define F_DECODE_CMD_RAW(cmd) (cmd >> F_TYPE_EXP)
+#define F_DECODE_CMD_TYPE(cmd) ((cmd) & ((1 << F_TYPE_EXP)-1))
+
+/* Encode small parameters into fcntl commands. This allows adding some flags
+   and other modifiers to fcntl commands without declaring a heap of new fcntl
+   commands variants. */
+#define F_ENCODE(cmd, small_param) (((cmd) & 0xFFFF) | ((small_param) << 16))
+#define F_DECODE_CMD(val) (val & 0xFFFF)
+#define F_DECODE_FLAGS(val) ((val & ~0xFFFF) >> 16)
+
+/* Set file descriptor status. */
+#define F_SETFD_NUM 0
+#define F_SETFD F_ENCODE_CMD(F_SETFD_NUM, F_TYPE_INT)
+
+/* Get file descriptor status. */
+#define F_GETFD_NUM 1
+#define F_GETFD F_ENCODE_CMD(F_GETFD_NUM, F_TYPE_VOID)
+
+/* Set descriptor table entry flags. */
+#define F_SETFL_NUM 3
+#define F_SETFL F_ENCODE_CMD(F_SETFL_NUM, F_TYPE_INT)
+
+/* Get descriptor table entry flags. */
+#define F_GETFL_NUM 3
+#define F_GETFL F_ENCODE_CMD(F_GETFL_NUM, F_TYPE_VOID)
 
 #define AT_FDCWD (-100)
 #define AT_REMOVEDIR (1<<0)

sortix/io.cpp

@@ -316,32 +316,27 @@ static int sys_fstat(int fd, struct stat* st)
 	return desc->stat(&ctx, st);
 }
 
-static int sys_fcntl(int fd, int cmd, unsigned long arg)
+static int sys_fcntl(int fd, int cmd, uintptr_t arg)
 {
+	// Operations on the descriptor table.
 	Ref<DescriptorTable> dtable = CurrentProcess()->GetDTable();
-	Ref<Descriptor> desc;
-	int ret = -1;
-	switch ( cmd )
-	{
-	case F_SETFD:
-		ret = dtable->SetFlags(fd, (int) arg) ? 0 : -1;
-		break;
-	case F_GETFD:
-		ret = dtable->GetFlags(fd);
-		break;
-	case F_SETFL:
-		if ( (desc = dtable->Get(fd)) )
-			ret = desc->SetFlags((int) arg) ? 0 : -1;
-		break;
-	case F_GETFL:
-		if ( (desc = dtable->Get(fd)) )
-			ret = desc->GetFlags();
-		break;
-	default:
-		errno = EINVAL;
-		break;
-	}
-	return ret;
+
+	// Operations on the file descriptior.
+	if ( cmd == F_SETFD )
+		return dtable->SetFlags(fd, (int) arg) ? 0 : -1;;
+	if ( cmd == F_GETFD )
+		return dtable->GetFlags(fd);
+
+	// Operations on the file description.
+	Ref<Descriptor> desc = dtable->Get(fd);
+	if ( !desc )
+		return -1;
+	if ( cmd == F_SETFL )
+		return desc->SetFlags((int) arg);
+	if ( cmd == F_GETFL )
+		return desc->GetFlags();
+
+	return errno = EINVAL, -1;
 }
 
 static int sys_ioctl(int fd, int cmd, void* /*ptr*/)