From a008a55ff680ec600b5617d6a1f1701450aa8911 Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Mon, 18 May 2020 14:56:34 +0100 Subject: [PATCH 1/3] Add a disabled builder for riscv64 emulated tests This will run all tests for `riscv64gc-unknown-linux-gnu` in a QEMU instance. This is based upon the armhf QEMU test image. --- src/bootstrap/configure.py | 2 + .../0001-Remove-stime-function-calls.patch | 96 +++++++++++++++++ .../disabled/riscv64gc-linux/Dockerfile | 102 ++++++++++++++++++ .../disabled/riscv64gc-linux/linux.config | 51 +++++++++ src/tools/remote-test-client/src/main.rs | 83 +++++++++++--- 5 files changed, 319 insertions(+), 15 deletions(-) create mode 100644 src/ci/docker/disabled/riscv64gc-linux/0001-Remove-stime-function-calls.patch create mode 100644 src/ci/docker/disabled/riscv64gc-linux/Dockerfile create mode 100644 src/ci/docker/disabled/riscv64gc-linux/linux.config diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index d1e53db573e..47673ce1e87 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -141,6 +141,8 @@ v("qemu-armhf-rootfs", "target.arm-unknown-linux-gnueabihf.qemu-rootfs", "rootfs in qemu testing, you probably don't want to use this") v("qemu-aarch64-rootfs", "target.aarch64-unknown-linux-gnu.qemu-rootfs", "rootfs in qemu testing, you probably don't want to use this") +v("qemu-riscv64-rootfs", "target.riscv64gc-unknown-linux-gnu.qemu-rootfs", + "rootfs in qemu testing, you probably don't want to use this") v("experimental-targets", "llvm.experimental-targets", "experimental LLVM targets to build") v("release-channel", "rust.channel", "the name of the release channel to build") diff --git a/src/ci/docker/disabled/riscv64gc-linux/0001-Remove-stime-function-calls.patch b/src/ci/docker/disabled/riscv64gc-linux/0001-Remove-stime-function-calls.patch new file mode 100644 index 00000000000..08d0c5b2cac --- /dev/null +++ b/src/ci/docker/disabled/riscv64gc-linux/0001-Remove-stime-function-calls.patch @@ -0,0 +1,96 @@ +From c820da85c65c7f3aa9e9cb3ed71ada69bf9b783e Mon Sep 17 00:00:00 2001 +From: Alistair Francis +Date: Tue, 19 Nov 2019 13:06:40 +0100 +Subject: [PATCH] Remove stime() function calls + +stime() has been deprecated in glibc 2.31 and replaced with +clock_settime(). Let's replace the stime() function calls with +clock_settime() in preperation. + +function old new delta +rdate_main 197 224 +27 +clock_settime - 27 +27 +date_main 926 941 +15 +stime 37 - -37 +------------------------------------------------------------------------------ +(add/remove: 2/2 grow/shrink: 2/0 up/down: 69/-37) Total: 32 bytes + +Signed-off-by: Alistair Francis +Signed-off-by: Denys Vlasenko + +[Tom Eccles: adjust patch context to apply on top of 1.31.1-stable] +Signed-off-by: Tom Eccles +--- + coreutils/date.c | 6 +++++- + libbb/missing_syscalls.c | 8 -------- + util-linux/rdate.c | 8 ++++++-- + 3 files changed, 11 insertions(+), 11 deletions(-) + +diff --git a/coreutils/date.c b/coreutils/date.c +index 3414d38ae..4ade6abb4 100644 +--- a/coreutils/date.c ++++ b/coreutils/date.c +@@ -279,6 +279,9 @@ int date_main(int argc UNUSED_PARAM, char **argv) + time(&ts.tv_sec); + #endif + } ++#if !ENABLE_FEATURE_DATE_NANO ++ ts.tv_nsec = 0; ++#endif + localtime_r(&ts.tv_sec, &tm_time); + + /* If date string is given, update tm_time, and maybe set date */ +@@ -301,9 +304,10 @@ int date_main(int argc UNUSED_PARAM, char **argv) + if (date_str[0] != '@') + tm_time.tm_isdst = -1; + ts.tv_sec = validate_tm_time(date_str, &tm_time); ++ ts.tv_nsec = 0; + + /* if setting time, set it */ +- if ((opt & OPT_SET) && stime(&ts.tv_sec) < 0) { ++ if ((opt & OPT_SET) && clock_settime(CLOCK_REALTIME, &ts) < 0) { + bb_perror_msg("can't set date"); + } + } +diff --git a/libbb/missing_syscalls.c b/libbb/missing_syscalls.c +index 87cf59b3d..dc40d9155 100644 +--- a/libbb/missing_syscalls.c ++++ b/libbb/missing_syscalls.c +@@ -15,14 +15,6 @@ pid_t getsid(pid_t pid) + return syscall(__NR_getsid, pid); + } + +-int stime(const time_t *t) +-{ +- struct timeval tv; +- tv.tv_sec = *t; +- tv.tv_usec = 0; +- return settimeofday(&tv, NULL); +-} +- + int sethostname(const char *name, size_t len) + { + return syscall(__NR_sethostname, name, len); +diff --git a/util-linux/rdate.c b/util-linux/rdate.c +index 70f829e7f..878375d78 100644 +--- a/util-linux/rdate.c ++++ b/util-linux/rdate.c +@@ -95,9 +95,13 @@ int rdate_main(int argc UNUSED_PARAM, char **argv) + if (!(flags & 2)) { /* no -p (-s may be present) */ + if (time(NULL) == remote_time) + bb_error_msg("current time matches remote time"); +- else +- if (stime(&remote_time) < 0) ++ else { ++ struct timespec ts; ++ ts.tv_sec = remote_time; ++ ts.tv_nsec = 0; ++ if (clock_settime(CLOCK_REALTIME, &ts) < 0) + bb_perror_msg_and_die("can't set time of day"); ++ } + } + + if (flags != 1) /* not lone -s */ +-- +2.25.1 + diff --git a/src/ci/docker/disabled/riscv64gc-linux/Dockerfile b/src/ci/docker/disabled/riscv64gc-linux/Dockerfile new file mode 100644 index 00000000000..f21dc2ba309 --- /dev/null +++ b/src/ci/docker/disabled/riscv64gc-linux/Dockerfile @@ -0,0 +1,102 @@ +# based on armhf-gnu/Dockerfile +FROM ubuntu:20.04 + +RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections +RUN apt-get update -y && apt-get install -y --no-install-recommends \ + bc \ + bison \ + ca-certificates \ + cmake \ + cpio \ + curl \ + debian-ports-archive-keyring \ + debootstrap \ + flex \ + gcc \ + gcc-riscv64-linux-gnu \ + git \ + g++-riscv64-linux-gnu \ + g++ \ + libc6-dev \ + libc6-dev-riscv64-cross \ + make \ + patch \ + python3 \ + qemu-system-misc \ + xz-utils + +ENV ARCH=riscv +ENV CROSS_COMPILE=riscv64-linux-gnu- + +WORKDIR /build + +# From https://github.com/michaeljclark/busybear-linux/blob/master/conf/linux.config +COPY riscv64gc-linux/linux.config /build + +# Compile the kernel that we're going to be emulating with. This is +# basically just done to be compatible with the QEMU target that we're going +# to be using when running tests. +RUN curl https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.6.16.tar.xz | tar xJf - && \ + cp linux.config linux-5.6.16/.config && \ + cd /build/linux-5.6.16 && \ + make olddefconfig && \ + make -j$(nproc) vmlinux +RUN cp linux-5.6.16/vmlinux /tmp +RUN rm -rf linux-5.6.16 + +# Compile an instance of busybox as this provides a lightweight system and init +# binary which we will boot into. Only trick here is configuring busybox to +# build static binaries. +RUN curl https://busybox.net/downloads/busybox-1.31.1.tar.bz2 | tar xjf - +COPY riscv64gc-linux/0001-Remove-stime-function-calls.patch /build/busybox-1.31.1/ +RUN cd /build/busybox-1.31.1 && \ + patch -p1 -i 0001-Remove-stime-function-calls.patch && \ + make defconfig && \ + sed -i 's/.*CONFIG_STATIC.*/CONFIG_STATIC=y/' .config && \ + make -j$(nproc) && \ + make install && \ + mv _install /tmp/rootfs && \ + cd /build && \ + rm -rf busybox-1.31.1 + +# Download the ubuntu rootfs, which we'll use as a chroot for all our tests +# This is only needed to provide /lib/* and /usr/lib/* +WORKDIR /tmp +RUN debootstrap --variant=minbase --arch=riscv64 --foreign focal /tmp/rootfs/ubuntu +RUN cd rootfs && mkdir proc sys dev etc etc/init.d +# rootfs/ubuntu/proc is in a weird state (access fails with ELOOP) until +# rootfs/ubuntu/debootstrap/debootstrap --second-stage is run (under emulation), +# but this takes ages. Instead hack it into a good enough state. +# /proc is used by std::env::current_exe() (which is roughly +# `readlink /proc/self/exe`) +RUN cd rootfs/ubuntu && rm -rf proc && mkdir proc + +# Copy over our init script, which starts up our test server and also a few other +# misc tasks +COPY scripts/qemu-bare-bones-rcS rootfs/etc/init.d/rcS +RUN chmod +x rootfs/etc/init.d/rcS + +# Helper to quickly fill the entropy pool in the kernel +COPY scripts/qemu-bare-bones-addentropy.c /tmp/addentropy.c +RUN riscv64-linux-gnu-gcc addentropy.c -o rootfs/addentropy -static + +# download and build the riscv bootloader +RUN git clone https://github.com/riscv/riscv-pk +WORKDIR /tmp/riscv-pk +# nothing special about this revision: it is just master at the time of writing +# v1.0.0 doesn't build +RUN git checkout 5d9ed238e1cabfbca3c47f50d32894ce94bfc304 +RUN mkdir build && cd build && \ + ../configure --with-payload=/tmp/vmlinux --host=riscv64-linux-gnu && \ + make -j$(nproc) && \ + cp bbl /tmp +WORKDIR /tmp +RUN rm -rf /tmp/riscv-pk + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +ENV RUST_CONFIGURE_ARGS --qemu-riscv64-rootfs=/tmp/rootfs +ENV SCRIPT python3 ../x.py test --target riscv64gc-unknown-linux-gnu + +ENV NO_CHANGE_USER=1 diff --git a/src/ci/docker/disabled/riscv64gc-linux/linux.config b/src/ci/docker/disabled/riscv64gc-linux/linux.config new file mode 100644 index 00000000000..5142664742f --- /dev/null +++ b/src/ci/docker/disabled/riscv64gc-linux/linux.config @@ -0,0 +1,51 @@ +CONFIG_DEFAULT_HOSTNAME="busybear" +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y +CONFIG_CGROUPS=y +CONFIG_CGROUP_SCHED=y +CONFIG_CFS_BANDWIDTH=y +CONFIG_CGROUP_BPF=y +CONFIG_NAMESPACES=y +CONFIG_USER_NS=y +CONFIG_CHECKPOINT_RESTORE=y +CONFIG_BLK_DEV_INITRD=y +CONFIG_EXPERT=y +CONFIG_BPF_SYSCALL=y +CONFIG_SMP=y +CONFIG_MODULES=y +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_PACKET_DIAG=y +CONFIG_UNIX=y +CONFIG_INET=y +CONFIG_NETLINK_DIAG=y +# CONFIG_WIRELESS is not set +CONFIG_PCI=y +CONFIG_DEVTMPFS=y +CONFIG_BLK_DEV_LOOP=y +CONFIG_VIRTIO_BLK=y +CONFIG_NETDEVICES=y +CONFIG_VIRTIO_NET=y +# CONFIG_ETHERNET is not set +# CONFIG_WLAN is not set +CONFIG_SERIAL_8250=y +CONFIG_SERIAL_8250_CONSOLE=y +CONFIG_SERIAL_OF_PLATFORM=y +CONFIG_HVC_RISCV_SBI=y +# CONFIG_HW_RANDOM is not set +# CONFIG_USB_SUPPORT is not set +CONFIG_VIRTIO_MMIO=y +CONFIG_SIFIVE_PLIC=y +CONFIG_RAS=y +CONFIG_EXT2_FS=y +CONFIG_EXT3_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_AUTOFS4_FS=y +CONFIG_MSDOS_FS=y +CONFIG_VFAT_FS=y +CONFIG_TMPFS=y +# CONFIG_CRYPTO_ECHAINIV is not set +# CONFIG_CRYPTO_HW is not set +CONFIG_PRINTK_TIME=y diff --git a/src/tools/remote-test-client/src/main.rs b/src/tools/remote-test-client/src/main.rs index efc29163455..299ec0f0b4c 100644 --- a/src/tools/remote-test-client/src/main.rs +++ b/src/tools/remote-test-client/src/main.rs @@ -107,13 +107,23 @@ fn start_android_emulator(server: &Path) { Command::new("adb").arg("shell").arg("/data/tmp/testd").spawn().unwrap(); } -fn start_qemu_emulator(target: &str, rootfs: &Path, server: &Path, tmpdir: &Path) { +fn prepare_rootfs(target: &str, rootfs: &Path, server: &Path, rootfs_img: &Path) { + t!(fs::copy(server, rootfs.join("testd"))); + + match target { + "arm-unknown-linux-gnueabihf" | "aarch64-unknown-linux-gnu" => { + prepare_rootfs_cpio(rootfs, rootfs_img) + } + "riscv64gc-unknown-linux-gnu" => prepare_rootfs_ext4(rootfs, rootfs_img), + _ => panic!("{} is not supported", target), + } +} + +fn prepare_rootfs_cpio(rootfs: &Path, rootfs_img: &Path) { // Generate a new rootfs image now that we've updated the test server // executable. This is the equivalent of: // // find $rootfs -print 0 | cpio --null -o --format=newc > rootfs.img - t!(fs::copy(server, rootfs.join("testd"))); - let rootfs_img = tmpdir.join("rootfs.img"); let mut cmd = Command::new("cpio"); cmd.arg("--null") .arg("-o") @@ -128,6 +138,38 @@ fn start_qemu_emulator(target: &str, rootfs: &Path, server: &Path, tmpdir: &Path t!(io::copy(&mut child.stdout.take().unwrap(), &mut t!(File::create(&rootfs_img)))); assert!(t!(child.wait()).success()); + fn add_files(w: &mut dyn Write, root: &Path, cur: &Path) { + for entry in t!(cur.read_dir()) { + let entry = t!(entry); + let path = entry.path(); + let to_print = path.strip_prefix(root).unwrap(); + t!(write!(w, "{}\u{0}", to_print.to_str().unwrap())); + if t!(entry.file_type()).is_dir() { + add_files(w, root, &path); + } + } + } +} + +fn prepare_rootfs_ext4(rootfs: &Path, rootfs_img: &Path) { + let mut dd = Command::new("dd"); + dd.arg("if=/dev/zero") + .arg(&format!("of={}", rootfs_img.to_string_lossy())) + .arg("bs=1M") + .arg("count=1024"); + let mut dd_child = t!(dd.spawn()); + assert!(t!(dd_child.wait()).success()); + + let mut mkfs = Command::new("mkfs.ext4"); + mkfs.arg("-d").arg(rootfs).arg(rootfs_img); + let mut mkfs_child = t!(mkfs.spawn()); + assert!(t!(mkfs_child.wait()).success()); +} + +fn start_qemu_emulator(target: &str, rootfs: &Path, server: &Path, tmpdir: &Path) { + let rootfs_img = &tmpdir.join("rootfs.img"); + prepare_rootfs(target, rootfs, server, rootfs_img); + // Start up the emulator, in the background match target { "arm-unknown-linux-gnueabihf" => { @@ -170,19 +212,30 @@ fn start_qemu_emulator(target: &str, rootfs: &Path, server: &Path, tmpdir: &Path .arg("virtio-net-device,netdev=net0,mac=00:00:00:00:00:00"); t!(cmd.spawn()); } - _ => panic!("cannot start emulator for: {}" < target), - } - - fn add_files(w: &mut dyn Write, root: &Path, cur: &Path) { - for entry in t!(cur.read_dir()) { - let entry = t!(entry); - let path = entry.path(); - let to_print = path.strip_prefix(root).unwrap(); - t!(write!(w, "{}\u{0}", to_print.to_str().unwrap())); - if t!(entry.file_type()).is_dir() { - add_files(w, root, &path); - } + "riscv64gc-unknown-linux-gnu" => { + let mut cmd = Command::new("qemu-system-riscv64"); + cmd.arg("-nographic") + .arg("-machine") + .arg("virt") + .arg("-m") + .arg("1024") + .arg("-bios") + .arg("none") + .arg("-kernel") + .arg("/tmp/bbl") + .arg("-append") + .arg("quiet console=ttyS0 root=/dev/vda rw") + .arg("-netdev") + .arg("user,id=net0,hostfwd=tcp::12345-:12345") + .arg("-device") + .arg("virtio-net-device,netdev=net0,mac=00:00:00:00:00:00") + .arg("-device") + .arg("virtio-blk-device,drive=hd0") + .arg("-drive") + .arg(&format!("file={},format=raw,id=hd0", &rootfs_img.to_string_lossy())); + t!(cmd.spawn()); } + _ => panic!("cannot start emulator for: {}" < target), } } From 253112d43c46f9c0c6c854dcc2c858741616deed Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Tue, 9 Jun 2020 15:44:31 +0100 Subject: [PATCH 2/3] Mark tests whcih don't work under riscv emulation --- src/libstd/sys/unix/process/process_common.rs | 1 + src/libstd/time.rs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstd/sys/unix/process/process_common.rs b/src/libstd/sys/unix/process/process_common.rs index 2d7267263de..6e33cdd3c48 100644 --- a/src/libstd/sys/unix/process/process_common.rs +++ b/src/libstd/sys/unix/process/process_common.rs @@ -428,6 +428,7 @@ mod tests { // ignored there. #[cfg_attr(target_arch = "arm", ignore)] #[cfg_attr(target_arch = "aarch64", ignore)] + #[cfg_attr(target_arch = "riscv64", ignore)] fn test_process_mask() { unsafe { // Test to make sure that a signal mask does not get inherited. diff --git a/src/libstd/time.rs b/src/libstd/time.rs index c36e78b1d00..9e4ecc9a80d 100644 --- a/src/libstd/time.rs +++ b/src/libstd/time.rs @@ -796,11 +796,11 @@ mod tests { // Right now for CI this test is run in an emulator, and apparently the // aarch64 emulator's sense of time is that we're still living in the - // 70s. + // 70s. This is also true for riscv (also qemu) // // Otherwise let's assume that we're all running computers later than // 2000. - if !cfg!(target_arch = "aarch64") { + if !cfg!(target_arch = "aarch64") && !cfg!(target_arch = "riscv64") { assert!(a > thirty_years); } From 086eaf8f69b736a7f4b6b8ee232540993d0c255d Mon Sep 17 00:00:00 2001 From: Tom Eccles Date: Wed, 10 Jun 2020 09:01:01 +0100 Subject: [PATCH 3/3] tools: remote-test-client: fix typo Thanks to jfrimmel for pointing this out Co-authored-by: J. Frimmel <31166235+jfrimmel@users.noreply.github.com> --- src/tools/remote-test-client/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/remote-test-client/src/main.rs b/src/tools/remote-test-client/src/main.rs index 299ec0f0b4c..1fafe109d34 100644 --- a/src/tools/remote-test-client/src/main.rs +++ b/src/tools/remote-test-client/src/main.rs @@ -235,7 +235,7 @@ fn start_qemu_emulator(target: &str, rootfs: &Path, server: &Path, tmpdir: &Path .arg(&format!("file={},format=raw,id=hd0", &rootfs_img.to_string_lossy())); t!(cmd.spawn()); } - _ => panic!("cannot start emulator for: {}" < target), + _ => panic!("cannot start emulator for: {}", target), } }