From ea58a5e299f0b47cf56148dd7fc98b7b3a56e89a Mon Sep 17 00:00:00 2001 From: Rafal Wojtczuk Date: Mon, 30 Aug 2010 11:40:19 +0200 Subject: [PATCH 1/7] Memory management across VMs, first release --- appvm/qubes_core | 1 + common/meminfo-writer | 4 ++++ rpm_spec/core-appvm.spec | 2 ++ 3 files changed, 7 insertions(+) create mode 100755 common/meminfo-writer diff --git a/appvm/qubes_core b/appvm/qubes_core index c8dd150..c291ec7 100755 --- a/appvm/qubes_core +++ b/appvm/qubes_core @@ -87,6 +87,7 @@ start() fi fi + /usr/lib/qubes/meminfo-writer & [ -x /rw/config/rc.local ] && /rw/config/rc.local success echo "" diff --git a/common/meminfo-writer b/common/meminfo-writer new file mode 100755 index 0000000..fdbfd29 --- /dev/null +++ b/common/meminfo-writer @@ -0,0 +1,4 @@ +#!/bin/sh +while sleep 1 ; do + xenstore-write memory/meminfo "`cat /proc/meminfo`" +done diff --git a/rpm_spec/core-appvm.spec b/rpm_spec/core-appvm.spec index a4444f0..2949a05 100644 --- a/rpm_spec/core-appvm.spec +++ b/rpm_spec/core-appvm.spec @@ -65,6 +65,7 @@ cp qubes_timestamp qvm-copy-to-vm qvm-open-in-dvm $RPM_BUILD_ROOT/usr/bin mkdir -p $RPM_BUILD_ROOT/usr/lib/qubes cp qubes_add_pendrive_script qubes_penctl qvm-copy-to-vm.kde $RPM_BUILD_ROOT/usr/lib/qubes ln -s /usr/bin/qvm-open-in-dvm $RPM_BUILD_ROOT/usr/lib/qubes/qvm-dvm-transfer +cp ../common/meminfo-writer $RPM_BUILD_ROOT/usr/lib/qubes mkdir -p $RPM_BUILD_ROOT/%{kde_service_dir} cp qvm-copy.desktop qvm-dvm.desktop $RPM_BUILD_ROOT/%{kde_service_dir} mkdir -p $RPM_BUILD_ROOT/etc/udev/rules.d @@ -187,6 +188,7 @@ rm -rf $RPM_BUILD_ROOT /usr/lib/qubes/qvm-copy-to-vm.kde %attr(4755,root,root) /usr/bin/qvm-open-in-dvm /usr/lib/qubes/qvm-dvm-transfer +/usr/lib/qubes/meminfo-writer %{kde_service_dir}/qvm-copy.desktop %{kde_service_dir}/qvm-dvm.desktop %attr(4755,root,root) /usr/lib/qubes/qubes_penctl From ebbefaabc2fbf3d9a54abfd9c7233f92ff7a8c25 Mon Sep 17 00:00:00 2001 From: Rafal Wojtczuk Date: Mon, 30 Aug 2010 11:43:30 +0200 Subject: [PATCH 2/7] Fix restore completion detection in appvm/qubes_core --- appvm/qubes_core | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/appvm/qubes_core b/appvm/qubes_core index c291ec7..1b7aa3f 100755 --- a/appvm/qubes_core +++ b/appvm/qubes_core @@ -35,13 +35,8 @@ start() (read a b c d ; xenstore-write device/qubes_used_mem $c) # we're still running in DispVM template echo "Waiting for save/restore..." - # WARNING: Nergalism! - # Apparently it has been determined that DomU kernel - # dmesg's "using vcpu" after restore - while ! dmesg -c | grep "using vcpu" ; do usleep 10 ; done - # we're now after restore in a new instance of a DispVM # ... wait until qubes_restore.c (in Dom0) recreates VM-specific keys - while ! xenstore-read qubes_vm_type 2>/dev/null ; do + while ! xenstore-read qubes_restore_complete 2>/dev/null ; do usleep 10 done echo Back to life. From 7ff498c43bb6e8b36c91f08a3e9d775cb746f9de Mon Sep 17 00:00:00 2001 From: Rafal Wojtczuk Date: Fri, 3 Sep 2010 16:23:09 +0200 Subject: [PATCH 3/7] qmemman: make meminfo-writer a C program --- common/Makefile | 7 +++++ common/meminfo-writer | 4 --- common/meminfo-writer.c | 62 ++++++++++++++++++++++++++++++++++++++++ rpm_spec/core-appvm.spec | 1 + 4 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 common/Makefile delete mode 100755 common/meminfo-writer create mode 100644 common/meminfo-writer.c diff --git a/common/Makefile b/common/Makefile new file mode 100644 index 0000000..0bef1ae --- /dev/null +++ b/common/Makefile @@ -0,0 +1,7 @@ +CC=gcc +CFLAGS=-Wall -g +all: meminfo-writer +meminfo-writer: meminfo-writer.o + $(CC) -g -o meminfo-writer meminfo-writer.o -lxenstore +clean: + rm -f meminfo-writer *.o *~ diff --git a/common/meminfo-writer b/common/meminfo-writer deleted file mode 100755 index fdbfd29..0000000 --- a/common/meminfo-writer +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -while sleep 1 ; do - xenstore-write memory/meminfo "`cat /proc/meminfo`" -done diff --git a/common/meminfo-writer.c b/common/meminfo-writer.c new file mode 100644 index 0000000..5e01903 --- /dev/null +++ b/common/meminfo-writer.c @@ -0,0 +1,62 @@ +/* + * The Qubes OS Project, http://www.qubes-os.org + * + * Copyright (C) 2010 Rafal Wojtczuk + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +int main() +{ + struct xs_handle *xs; + int fd, n; + char buf[4096]; + + openlog("meminfo-writer", LOG_CONS | LOG_PID, LOG_DAEMON); + xs = xs_domain_open(); + if (!xs) { + syslog(LOG_DAEMON | LOG_ERR, "xs_domain_open"); + exit(1); + } + for (;;) { + fd = open("/proc/meminfo", O_RDONLY); + if (fd < 0) { + syslog(LOG_DAEMON | LOG_ERR, + "error opening /proc/meminfo ?"); + exit(1); + } + n = read(fd, buf, sizeof(buf)); + if (n <= 0) { + syslog(LOG_DAEMON | LOG_ERR, + "error reading /proc/meminfo ?"); + exit(1); + } + close(fd); + if (!xs_write(xs, XBT_NULL, "memory/meminfo", buf, n)) { + syslog(LOG_DAEMON | LOG_ERR, + "error writing xenstore ?"); + exit(1); + } + sleep(1); + } +} diff --git a/rpm_spec/core-appvm.spec b/rpm_spec/core-appvm.spec index 2949a05..7974e23 100644 --- a/rpm_spec/core-appvm.spec +++ b/rpm_spec/core-appvm.spec @@ -52,6 +52,7 @@ fi %build make clean all +make -C ../common %install From 555be9b8577c41c462957ba728b82224fa5ee565 Mon Sep 17 00:00:00 2001 From: Rafal Wojtczuk Date: Thu, 9 Sep 2010 17:51:53 +0200 Subject: [PATCH 4/7] qmemman: offload some processing to meminfo-writer Make meminfo-writer compute used memory, and report to qmemman only if it has changed significantly enough. As it is written in C, its code is much faster that qmemman-server; also in the idle case, it saves on xenstore communication overhead. Allows to send updates up to 10 times per second, with CPU load on the VM below 0.1%. --- appvm/qubes_core | 5 +- common/meminfo-writer.c | 156 +++++++++++++++++++++++++++------------- 2 files changed, 111 insertions(+), 50 deletions(-) diff --git a/appvm/qubes_core b/appvm/qubes_core index 1b7aa3f..aeaf53c 100755 --- a/appvm/qubes_core +++ b/appvm/qubes_core @@ -82,7 +82,10 @@ start() fi fi - /usr/lib/qubes/meminfo-writer & + MEM_CHANGE_THRESHOLD_KB=30000 + MEMINFO_DELAY_USEC=100000 + /usr/lib/qubes/meminfo-writer $MEM_CHANGE_THRESHOLD_KB $MEMINFO_DELAY_USEC & + [ -x /rw/config/rc.local ] && /rw/config/rc.local success echo "" diff --git a/common/meminfo-writer.c b/common/meminfo-writer.c index 5e01903..d97c68d 100644 --- a/common/meminfo-writer.c +++ b/common/meminfo-writer.c @@ -1,62 +1,120 @@ -/* - * The Qubes OS Project, http://www.qubes-os.org - * - * Copyright (C) 2010 Rafal Wojtczuk - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * - */ -#include -#include -#include -#include #include +#include +#include #include #include #include -int main() -{ - struct xs_handle *xs; - int fd, n; - char buf[4096]; +#include - openlog("meminfo-writer", LOG_CONS | LOG_PID, LOG_DAEMON); +unsigned long prev_used_mem; +int used_mem_change_threshold; +int delay; + +char *parse(char *buf) +{ + char *ptr = buf; + char name[256]; + static char outbuf[4096]; + int val; + int len; + int MemTotal=0, MemFree=0, Buffers=0, Cached=0, SwapTotal=0, SwapFree=0; + unsigned long long key; + long used_mem, used_mem_diff; + int nitems = 0; + + while (nitems != 6) { + sscanf(ptr, "%s %d kB\n%n", name, &val, &len); + key = *(unsigned long long *) ptr; + if (key == *(unsigned long long *) "MemTotal:") { + MemTotal = val; + nitems++; + } else if (key == *(unsigned long long *) "MemFree:") { + MemFree = val; + nitems++; + } else if (key == *(unsigned long long *) "Buffers:") { + Buffers = val; + nitems++; + } else if (key == *(unsigned long long *) "Cached: ") { + Cached = val; + nitems++; + } else if (key == *(unsigned long long *) "SwapTotal:") { + SwapTotal = val; + nitems++; + } else if (key == *(unsigned long long *) "SwapFree:") { + SwapFree = val; + nitems++; + } + + ptr += len; + } + + used_mem = + MemTotal - Buffers - Cached - MemFree + SwapTotal - SwapFree; + if (used_mem < 0) + return NULL; + + used_mem_diff = used_mem - prev_used_mem; + prev_used_mem = used_mem; + if (used_mem_diff < 0) + used_mem_diff = -used_mem_diff; + if (used_mem_diff > used_mem_change_threshold) { + sprintf(outbuf, + "MemTotal: %d kB\nMemFree: %d kB\nBuffers: %d kB\nCached: %d kB\n" + "SwapTotal: %d kB\nSwapFree: %d kB\n", MemTotal, + MemFree, Buffers, Cached, SwapTotal, SwapFree); + return outbuf; + } + return NULL; +} + +void usage() +{ + fprintf(stderr, + "usage: meminfo_writer threshold_in_kb delay_in_us\n"); + exit(1); +} + +void send_to_qmemman(struct xs_handle *xs, char *data) +{ + if (!xs_write(xs, XBT_NULL, "memory/meminfo", data, strlen(data))) { + syslog(LOG_DAEMON | LOG_ERR, "error writing xenstore ?"); + exit(1); + } +} + +int +main(int argc, char **argv) +{ + char buf[4096]; + int n; + char *meminfo_data; + int fd; + struct xs_handle *xs; + + if (argc != 3) + usage(); + used_mem_change_threshold = atoi(argv[1]); + delay = atoi(argv[2]); + if (!used_mem_change_threshold || !delay) + usage(); + + fd = open("/proc/meminfo", O_RDONLY); + if (fd < 0) { + perror("open meminfo"); + exit(1); + } xs = xs_domain_open(); if (!xs) { - syslog(LOG_DAEMON | LOG_ERR, "xs_domain_open"); + perror("xs_domain_open"); exit(1); } for (;;) { - fd = open("/proc/meminfo", O_RDONLY); - if (fd < 0) { - syslog(LOG_DAEMON | LOG_ERR, - "error opening /proc/meminfo ?"); - exit(1); - } n = read(fd, buf, sizeof(buf)); - if (n <= 0) { - syslog(LOG_DAEMON | LOG_ERR, - "error reading /proc/meminfo ?"); - exit(1); - } - close(fd); - if (!xs_write(xs, XBT_NULL, "memory/meminfo", buf, n)) { - syslog(LOG_DAEMON | LOG_ERR, - "error writing xenstore ?"); - exit(1); - } - sleep(1); + buf[n] = 0; + meminfo_data = parse(buf); + if (meminfo_data) + send_to_qmemman(xs, meminfo_data); + usleep(delay); + lseek(fd, 0, SEEK_SET); } } From 02b8e51341fe20a4057cc027536e62c5055fc1ef Mon Sep 17 00:00:00 2001 From: Rafal Wojtczuk Date: Fri, 10 Sep 2010 11:35:30 +0200 Subject: [PATCH 5/7] qmemman: when a AppVM is low on memory, allow small adjustments A small AppVM (say, with 100MB total) can go below prefmem, and still not be assigned memory, because of the MIN_TOTAL_MEMORY_TRANSFER threshold. So, if AppVM is below prefmem, allow for smaller mem-sets. --- common/meminfo-writer.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/common/meminfo-writer.c b/common/meminfo-writer.c index d97c68d..2f6f217 100644 --- a/common/meminfo-writer.c +++ b/common/meminfo-writer.c @@ -17,7 +17,8 @@ char *parse(char *buf) static char outbuf[4096]; int val; int len; - int MemTotal=0, MemFree=0, Buffers=0, Cached=0, SwapTotal=0, SwapFree=0; + int MemTotal = 0, MemFree = 0, Buffers = 0, Cached = 0, SwapTotal = + 0, SwapFree = 0; unsigned long long key; long used_mem, used_mem_diff; int nitems = 0; @@ -54,10 +55,12 @@ char *parse(char *buf) return NULL; used_mem_diff = used_mem - prev_used_mem; - prev_used_mem = used_mem; if (used_mem_diff < 0) used_mem_diff = -used_mem_diff; - if (used_mem_diff > used_mem_change_threshold) { + if (used_mem_diff > used_mem_change_threshold + || (used_mem > prev_used_mem && used_mem * 13 / 10 > MemTotal + && used_mem_diff > used_mem_change_threshold/2)) { + prev_used_mem = used_mem; sprintf(outbuf, "MemTotal: %d kB\nMemFree: %d kB\nBuffers: %d kB\nCached: %d kB\n" "SwapTotal: %d kB\nSwapFree: %d kB\n", MemTotal, @@ -82,8 +85,7 @@ void send_to_qmemman(struct xs_handle *xs, char *data) } } -int -main(int argc, char **argv) +int main(int argc, char **argv) { char buf[4096]; int n; From a2136c3d2bc61d75869418f05e416e511ab22072 Mon Sep 17 00:00:00 2001 From: Rafal Wojtczuk Date: Fri, 10 Sep 2010 11:38:06 +0200 Subject: [PATCH 6/7] Compile meminfo-writer with -O3 --- common/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/Makefile b/common/Makefile index 0bef1ae..85888a9 100644 --- a/common/Makefile +++ b/common/Makefile @@ -1,5 +1,5 @@ CC=gcc -CFLAGS=-Wall -g +CFLAGS=-Wall -g -O3 all: meminfo-writer meminfo-writer: meminfo-writer.o $(CC) -g -o meminfo-writer meminfo-writer.o -lxenstore From 6d764ef50d5af47ac749801e876589e5d909a5f1 Mon Sep 17 00:00:00 2001 From: Rafal Wojtczuk Date: Fri, 10 Sep 2010 14:53:41 +0200 Subject: [PATCH 7/7] qmemman: save a syscall in meminfo-writer via use of "pread" --- common/meminfo-writer.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/meminfo-writer.c b/common/meminfo-writer.c index 2f6f217..49c8b6a 100644 --- a/common/meminfo-writer.c +++ b/common/meminfo-writer.c @@ -111,12 +111,11 @@ int main(int argc, char **argv) exit(1); } for (;;) { - n = read(fd, buf, sizeof(buf)); + n = pread(fd, buf, sizeof(buf), 0); buf[n] = 0; meminfo_data = parse(buf); if (meminfo_data) send_to_qmemman(xs, meminfo_data); usleep(delay); - lseek(fd, 0, SEEK_SET); } }