feat(libkernel): support ioctl on non-socket descriptors

ioctl and _ioctl answered EBADF for every descriptor outside the
libSceNet socket table. Decoded call sites: PS5MC (PPSA99420, FIONBIO x3,
FIONREAD x2, TIOCGWINSZ x3), ProsperoTV (PPSA99003, FIONBIO on a pipe
once poll on pipes is in), XPSemu (PPSA97358, 21 sites).

Following FreeBSD 11.4 kern_ioctl (sys/kern/sys_generic.c:758),
vn_ioctl (sys/kern/vfs_vnops.c:1488), pipe_ioctl (sys/kern/sys_pipe.c:1346)
and randomdev_ioctl (sys/dev/random/randomdev.c:301), release/11.4.0
8c639945: FIONBIO sets or clears O_NONBLOCK through the same status
flags as fcntl F_SETFL (EFAULT for a null argument before EBADF);
FIONREAD reports the size past the offset of a file, the queued bytes of
a pipe's read end, 0 on its write end, ENOTTY on the random device, and
throws on a directory or another character device; termios requests
(group 't', TIOCGWINSZ included) fail with ENOTTY, as tcgetattr already
takes these descriptors as not a terminal; any other request throws
naming the request and descriptor kind. Sockets keep their path
unchanged.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

core/libs/prx/libkernel/File/include/DescriptorControl.hpp

@@ -6,6 +6,7 @@
 namespace File {
 
 int Fcntl(int fd, int command, std::intptr_t argument);
+int Ioctl(int fd, std::uint64_t request, void* argument);
 int DuplicateFrom(int fd, int start);
 void RememberDescriptor(int fd, int openFlags);
 void ShareDescriptor(int from, int to);

core/libs/prx/libkernel/File/src/DescriptorControl.cpp

@@ -7,6 +7,7 @@
 
 #include <cerrno>
 #include <cstddef>
+#include <cstdio>
 #include <cstring>
 #include <map>
 #include <memory>
@@ -21,6 +22,7 @@
 #include <winternl.h>
 #else
 #include <fcntl.h>
+#include <sys/ioctl.h>
 #include <sys/stat.h>
 #include <unistd.h>
 #endif
@@ -83,6 +85,11 @@ constexpr std::int16_t Unlock = 2;
 constexpr std::int16_t WriteLock = 3;
 constexpr std::int16_t FromCurrent = 1;
 
+constexpr std::uint32_t NonblockingRequest = 0x8004667e;
+constexpr std::uint32_t ReadableRequest = 0x4004667f;
+constexpr std::uint32_t RequestIn = 0x80000000u;
+constexpr std::uint32_t TerminalGroup = 't';
+
 struct GuestLock {
     std::int64_t start;
     std::int64_t length;
@@ -187,6 +194,23 @@ void ApplyHostStatus(int fd, Kind kind, int changed, int next, const char* funct
     if ((changed & Nonblocking) && kind == Kind::Pipe) SetPipeNonblocking(fd, (next & Nonblocking) != 0, function);
 }
 
+int Readable(int fd, Kind kind, const char* function) {
+    if (kind == Kind::File) {
+        const auto size = ::_filelengthi64(fd);
+        const auto offset = ::_telli64(fd);
+        if (size < 0 || offset < 0)
+            throw std::runtime_error(std::string(function) + ": cannot read the size and offset of descriptor " + std::to_string(fd) + ", errno=" + std::to_string(errno));
+        return static_cast<int>(size - offset);
+    }
+    if (AccessMode(fd, function) == WriteOnly) return 0;
+    DWORD available = 0;
+    if (!::PeekNamedPipe(NativeHandle(fd), nullptr, 0, nullptr, &available, nullptr)) {
+        const DWORD error = ::GetLastError();
+        if (error == ERROR_BROKEN_PIPE) return 0;
+        throw std::runtime_error(std::string(function) + ": cannot peek pipe descriptor " + std::to_string(fd) + ", error=" + std::to_string(error));
+    }
+    return static_cast<int>(available);
+}
 #else
 std::optional<Kind> Classify(int fd, const char* function) {
     if (fd < 0) return std::nullopt;
@@ -234,6 +258,20 @@ void ApplyHostStatus(int fd, Kind, int changed, int next, const char* function)
         throw std::runtime_error(std::string(function) + ": host F_SETFL failed, fd=" + std::to_string(fd) + ", errno=" + std::to_string(errno));
 }
 
+int Readable(int fd, Kind kind, const char* function) {
+    if (kind == Kind::File) {
+        struct stat status{};
+        const auto offset = ::lseek(fd, 0, SEEK_CUR);
+        if (::fstat(fd, &status) != 0 || offset < 0)
+            throw std::runtime_error(std::string(function) + ": cannot read the size and offset of descriptor " + std::to_string(fd) + ", errno=" + std::to_string(errno));
+        return static_cast<int>(status.st_size - offset);
+    }
+    if (AccessMode(fd, function) == WriteOnly) return 0;
+    int available = 0;
+    if (::ioctl(fd, FIONREAD, &available) != 0)
+        throw std::runtime_error(std::string(function) + ": host FIONREAD failed, fd=" + std::to_string(fd) + ", errno=" + std::to_string(errno));
+    return available;
+}
 
 int LockError(int error) {
     switch (error) {
@@ -402,6 +440,32 @@ int Fcntl(int fd, int command, std::intptr_t argument) {
     }
 }
 
+int Ioctl(int fd, std::uint64_t request, void* argument) {
+    const auto command = static_cast<std::uint32_t>(request);
+    const std::uint32_t length = (command >> 16) & 0x1fff;
+    if ((command & RequestIn) && length != 0 && argument == nullptr) return Fail(GuestEfault);
+    const auto kind = Classify(fd, "ioctl");
+    if (!kind) return Fail(GuestEbadf);
+    if (command == NonblockingRequest) {
+        const bool enable = *static_cast<const int*>(argument) != 0;
+        return ChangeStatus(fd, *kind, Nonblocking, enable ? Nonblocking : 0, "ioctl");
+    }
+    if (command == ReadableRequest) {
+        if (*kind == Kind::RandomDevice) return Fail(GuestEnotty);
+        if (*kind != Kind::File && *kind != Kind::Pipe) Unsupported("ioctl", "FIONREAD", *kind, fd);
+        const int available = Readable(fd, *kind, "ioctl");
+        if (argument == nullptr) return Fail(GuestEfault);
+        const GuestArena::HostWrite destination(argument, sizeof(available));
+        if (!destination.Open()) return Fail(GuestEfault);
+        std::memcpy(argument, &available, sizeof(available));
+        return 0;
+    }
+    if (((command >> 8) & 0xff) == TerminalGroup) return Fail(GuestEnotty);
+    char hex[11];
+    std::snprintf(hex, sizeof(hex), "0x%08X", command);
+    Unsupported("ioctl", std::string("request ") + hex, *kind, fd);
+}
+
 void RememberDescriptor(int fd, int openFlags) {
     std::lock_guard lock(g_mutex);
     g_entries[fd] = Entry{(openFlags & OpenCloseOnExec) != 0, std::make_shared<int>(openFlags & OpenStatusFlags)};

core/libs/prx/libkernel/Socket/src/Udp.cpp

@@ -665,6 +665,7 @@ int APS5_VABI getpeername_nid_postfix(int descriptor, void* address, std::uint32
     return 0;
 }
 int APS5_VABI ioctl_nid_postfix(int descriptor, std::uint64_t request, void* argument) {
+    if (descriptor < GuestSockets::FirstDescriptor) return File::Ioctl(descriptor, request, argument);
     const auto socket = Lookup(descriptor);
     if (!socket) return -1;
     if (!argument) return Fail(14);

core/libs/tests/GuestKernelFcntl.cpp

@@ -41,8 +41,12 @@ static constexpr int CreateWriteAppend = 0x209;
 static constexpr int GuestEbadf = 9;
 static constexpr int GuestEfault = 14;
 static constexpr int GuestEinval = 22;
+static constexpr int GuestEnotty = 25;
 static constexpr int GuestEagain = 35;
+static constexpr std::uint64_t NonblockingRequest = 0x8004667e;
 static constexpr std::uint64_t ReadableRequest = 0x4004667f;
+static constexpr std::uint64_t WindowSizeRequest = 0x40087468;
+static constexpr std::uint64_t AsyncRequest = 0x8004667d;
 static constexpr int SceEbadf = static_cast<int>(0x80020009u);
 static constexpr int SceEinval = static_cast<int>(0x80020016u);
 static constexpr int SceEnotsup = static_cast<int>(0x8002002du);
@@ -118,6 +122,12 @@ static void FileFlagsRoundTrip() {
 
     Require(write_nid_postfix(file, "abcde", 5) == 5);
     Require(sceKernelLseek(file, 2, 0) == 2);
+    int readable = -1;
+    Require(ioctl_nid_postfix(file, ReadableRequest, &readable) == 0 && readable == 3);
+    char size[8] = {};
+    RequireFailure(ioctl_nid_postfix(file, WindowSizeRequest, size), GuestEnotty);
+    RequireFailure(ioctl_nid_postfix(file, NonblockingRequest, nullptr), GuestEfault);
+    Require(ThrowsWith([file] { int on = 1; ioctl_nid_postfix(file, AsyncRequest, &on); }, "ioctl: request 0x8004667D on a file is not implemented"));
 
     RequireFailure(fcntl_nid_postfix(file, DuplicateCommand, -1), GuestEinval);
     Require(sceKernelFcntl(file, DuplicateCommand, -1) == SceEinval);
@@ -140,6 +150,7 @@ static void FileFlagsRoundTrip() {
     for (const int descriptor : {closing, duplicate, file}) Require(close_nid_postfix(descriptor) == 0);
     RequireFailure(fcntl_nid_postfix(file, GetDescriptorFlags), GuestEbadf);
     RequireFailure(fcntl_nid_postfix(file, DuplicateCommand, 0), GuestEbadf);
+    RequireFailure(ioctl_nid_postfix(file, ReadableRequest, &readable), GuestEbadf);
     Require(sceKernelFcntl(file, GetFlags) == SceEbadf);
 
     const int appending = sceKernelOpen("kernel_fcntl_probe.bin", CreateWriteAppend, 0644);
@@ -172,9 +183,19 @@ static void PipeFlagsRoundTrip() {
     char buffer[8] = {};
     RequireFailure(static_cast<int>(read_nid_postfix(pipe[0], buffer, sizeof(buffer))), GuestEagain);
     Require(write_nid_postfix(pipe[1], "abc", 3) == 3);
+    int readable = -1;
+    Require(ioctl_nid_postfix(pipe[0], ReadableRequest, &readable) == 0 && readable == 3);
+    Require(ioctl_nid_postfix(pipe[1], ReadableRequest, &readable) == 0 && readable == 0);
     Require(read_nid_postfix(pipe[0], buffer, sizeof(buffer)) == 3);
     RequireFailure(static_cast<int>(read_nid_postfix(pipe[0], buffer, sizeof(buffer))), GuestEagain);
 
+    int off = 0;
+    Require(ioctl_nid_postfix(pipe[0], NonblockingRequest, &off) == 0);
+    Require(fcntl_nid_postfix(pipe[0], GetFlags) == ReadOnly);
+    int on = 1;
+    Require(ioctl_nid_postfix(pipe[0], NonblockingRequest, &on) == 0);
+    Require(fcntl_nid_postfix(pipe[0], GetFlags) == (ReadOnly | NonBlocking));
+
     Require(fcntl_nid_postfix(pipe[1], SetFlags, NonBlocking) == 0);
     char block[1024] = {};
     std::int64_t written = 0;
@@ -186,6 +207,8 @@ static void PipeFlagsRoundTrip() {
     RequireFailure(fcntl_nid_postfix(pipe[0], DarwinReadAhead, 1), GuestEbadf);
     GuestLock lock{0, 0, 0, 3, 0, 0};
     RequireFailure(fcntl_nid_postfix(pipe[1], SetLock, &lock), GuestEbadf);
+    char size[8] = {};
+    RequireFailure(ioctl_nid_postfix(pipe[0], WindowSizeRequest, size), GuestEnotty);
     Require(ThrowsWith([&pipe] { fcntl_nid_postfix(pipe[0], SetFlags, NonBlocking | 0x40); }, "fcntl: O_ASYNC on a pipe is not implemented"));
     for (const int descriptor : pipe) Require(close_nid_postfix(descriptor) == 0);
 }

docs/dev/TechnicalDebt.md

@@ -654,7 +654,7 @@ Throughout the project, every function at every stage either **does exactly what
 - [kill](../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) follows FreeBSD's `kern_kill` and `killpg1` for the only guest process: `EINVAL` for signals outside 0 to 128, signal 0 only checks that the target exists, and its own pid, 0 and minus its own pid (the guest is taken to lead its own process group) signal it through `raise`, so they share `raise`'s limits above. Any other pid, and -1, which skips the caller, report `ESRCH`, so `EPERM` is never reported
 - [sceKernelFcntl](../../core/libs/prx/libkernel/File/src/Open.cpp) (libkernel) runs the POSIX `fcntl` below and returns its guest errno as `SCE_KERNEL_ERROR_*`, the form the other `sceKernel` file exports use. The return values and error codes are unverified on a console
 - [fcntl](../../core/libs/prx/libkernel/File/src/DescriptorControl.cpp) (libkernel) follows FreeBSD 11.4's [`kern_fcntl`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/kern/kern_descrip.c#L479-L799) and [`kern_dup`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/kern/kern_descrip.c#L812) on file, directory, pipe and character device descriptors; a socket keeps the libSceNet path (`F_GETFL`, `F_SETFL` with `O_NONBLOCK`, `EINVAL` for every other command, `F_GETFD` and `F_SETFD` included). The close-on-exec flag and the status flags live in a guest table, the status flags shared by `dup`, `dup2` and `F_DUPFD` as in an open file description; a descriptor the table has not seen, such as 0 to 2, starts from the host's flags, since the console's are not known. `F_GETFL` takes the access mode from the host (the granted access of the handle on Windows), so pipe ends report `O_RDONLY` and `O_WRONLY` where FreeBSD's bidirectional pipes report `O_RDWR`. `F_DUPFD` and `F_DUPFD_CLOEXEC` return the lowest free host descriptor at or above the start, so the bound is the host table (`EMFILE` on Windows, `EINVAL` past `RLIMIT_NOFILE` on Linux), not the console's unknown `RLIMIT_NOFILE` and `kern.maxfilesperproc`. `O_NONBLOCK` switches a host pipe (`PIPE_NOWAIT` on Windows, where an empty read and a full write then report `EAGAIN`) and is only recorded on files, directories and the random device, as their reads never block; on a Windows console or other character device it throws. Changing `O_APPEND` on a Windows file, changing `O_FSYNC` on a file, `O_ASYNC` on a pipe or character device (`SIGIO` is not delivered), `F_GETOWN`, `F_SETOWN` and `F_SETLK_REMOTE` throw; `O_DIRECT`, `F_RDAHEAD` and `F_READAHEAD` are recorded as the hints they are. Record locks (`F_GETLK`, `F_SETLK`, `F_SETLKW` and their forms without `l_sysid`) use the host's POSIX locks on Linux and throw on Windows, and `F_GETLK` with `F_UNLCK` throws. The return values and error codes are unverified on a console
-- [ioctl, _ioctl](../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) handle only `FIONBIO` and `FIONREAD` on a socket. Any other request on a socket fails with `ENOTTY`, and any request on a file or pipe descriptor fails with `EBADF`, where FreeBSD answers `FIONBIO` and `FIONREAD` there too and fails other requests, termios ones included, with `ENOTTY`. Stremio Plus (PPSA74126) imports `_ioctl` for its statically linked `tc*` functions, which send `TIOCGETA`, `TIOCSETA`, `TIOCSPGRP` and the other termios requests
+- [ioctl, _ioctl](../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) handle only `FIONBIO` and `FIONREAD` on a socket, and any other request there fails with `ENOTTY`. On the other descriptors they follow FreeBSD's [`kern_ioctl`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/kern/sys_generic.c#L758), [`vn_ioctl`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/kern/vfs_vnops.c#L1488) and [`pipe_ioctl`](https://github.com/freebsd/freebsd-src/blob/8c639945a44588f6cfdddfb057bb5aad88735da0/sys/kern/sys_pipe.c#L1346): `FIONBIO` sets `O_NONBLOCK` as `fcntl` does, and `FIONREAD` reports the size past the offset of a file, the queued bytes of a pipe's read end, 0 on its write end and `ENOTTY` on the random device, while it throws on a directory or another character device. Every termios request (group `t`, `TIOCGWINSZ` included) fails with `ENOTTY`, since those descriptors are taken as not a terminal, as in `tcgetattr`, and any other request throws. Stremio Plus (PPSA74126) imports `_ioctl` for its statically linked `tc*` functions, which send `TIOCGETA`, `TIOCSETA`, `TIOCSPGRP` and the other termios requests
 - [sendfile](../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) follows FreeBSD 11's `sendfile` for a connected stream socket: headers, the file from `offset` (to its end when `nbytes` is 0) and trailers, the file offset left unchanged, `EAGAIN` with `*sbytes` set after a partial send on a non-blocking socket. It reads the file through `sceKernelPread` in 64 KiB blocks and sends each with `send`. A socket as the file fails with `EINVAL`, as on FreeBSD, but any other descriptor `sceKernelPread` reads, such as `/dev/urandom`, is sent where FreeBSD requires a regular file or shared memory, and a file descriptor as the socket fails with `EBADF`, as for `send`, where FreeBSD answers `ENOTSOCK`. `flags` are ignored: reads complete before sending, so `SF_NODISKIO` never fails with `EBUSY`, and `SF_SYNC` returns once the data is queued on the socket rather than sent. PS5MC (PPSA99420) imports it
 - [pthread_kill](../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) returns the error number like FreeBSD's `pthread_kill`: `EINVAL` for signals outside 0 to 128 and `ESRCH` for a null thread. Signal 0 reports any other thread handle as existing, without checking that the thread is still alive. A signal to the calling thread goes through `raise`, so it shares `raise`'s limits; a signal to another thread throws, since guest signals are not delivered to a chosen thread
 - [select](../../core/libs/prx/libkernel/Socket/src/Socket.cpp) (libkernel) validates `tv_usec` but saturates a valid timeout beyond the remaining `steady_clock` range at `time_point::max()`, so the wait can continue effectively indefinitely. The console's behavior for a valid timeout outside the host clock's range is unknown.