1

Running on Ubuntu server, a CentOS 7 VM is eating the memory :

Host:

# free -m
              total        used        free      shared  buff/cache   available
Mem:          80140       71676        5346           7        3117        7655
Swap:          4095         265        3830

# ps_mem
 Private  +   Shared  =  RAM used   Program

 59.7 MiB +  67.4 MiB = 127.1 MiB   fail2ban-server (4)
189.2 MiB + 189.6 MiB = 378.8 MiB   mysqld
202.2 MiB + 206.9 MiB = 409.1 MiB   lxd (5)
128.5 MiB + 558.5 MiB = 687.0 MiB   systemd-journald (5)
 30.1 GiB +  30.3 GiB =  60.4 GiB   qemu-system-x86_64
---------------------------------
                         62.8 GiB
=================================

# virsh dommemstat ispmanager-master
actual 33554432
swap_in 1214268
swap_out 6718144
major_fault 429115
minor_fault 27183560666
unused 569424
available 32779288
usable 19841256
last_update 1597820230
rss 33239928

Guest:

# free -m
              total        used        free      shared  buff/cache   available
Mem:          32011       11235         548         978       20226       19403
Swap:         10235        2568        7667

# lsmod | grep virt
virtio_balloon         18015  0 
virtio_net             28114  0 
net_failover           18147  1 virtio_net
virtio_blk             18323  4 
virtio_console         28076  1 
virtio_pci             22985  0 
virtio_ring            22952  5 virtio_blk,virtio_net,virtio_pci,virtio_balloon,virtio_console
virtio                 14959  5 virtio_blk,virtio_net,virtio_pci,virtio_balloon,virtio_consol

# virsh dommemstat ispmanager-master
actual 33554432
swap_in 0
swap_out 168
major_fault 24275
minor_fault 796131035
unused 395204
available 32779288
usable 25097908
last_update 1597885620
rss 33535884

The guest config:

<domain type="kvm">
  <name>ispmanager-master</name>
  <uuid>353f2014-53d7-413e-b635-60eb4e78751b</uuid>
  <metadata>
    <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
      <libosinfo:os id="http://centos.org/centos/7.0"/>
    </libosinfo:libosinfo>
  </metadata>
  <memory unit="KiB">33554432</memory>
  <currentMemory unit="KiB">33554432</currentMemory>
  <vcpu placement="static">10</vcpu>
  <os>
    <type arch="x86_64" machine="pc-i440fx-bionic">hvm</type>
  </os>
  <features>
    <acpi/>
    <apic/>
    <vmport state="off"/>
  </features>
  <cpu mode="custom" match="exact" check="partial">
    <model fallback="allow">Skylake-Server-IBRS</model>
  </cpu>
  <clock offset="utc">
    <timer name="rtc" tickpolicy="catchup"/>
    <timer name="pit" tickpolicy="delay"/>
    <timer name="hpet" present="no"/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <pm>
    <suspend-to-mem enabled="no"/>
    <suspend-to-disk enabled="no"/>
  </pm>
  <devices>
    <emulator>/usr/bin/kvm-spice</emulator>
    <disk type="block" device="disk">
      <driver name="qemu" type="raw" cache="none" io="native"/>
      <source dev="/dev/zvol/pool1/vm/isp-master"/>
      <target dev="vda" bus="virtio"/>
      <boot order="1"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x07" function="0x0"/>
    </disk>
    <disk type="block" device="disk">
      <driver name="qemu" type="raw" cache="none" io="native"/>
      <source dev="/dev/zvol/pool1/vm/isp-master-backups"/>
      <target dev="vdb" bus="virtio"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x09" function="0x0"/>
    </disk>
    <disk type="block" device="disk">
      <driver name="qemu" type="raw" cache="none" io="native"/>
      <source dev="/dev/zvol/ssd1/vm/isp-master"/>
      <target dev="vdc" bus="virtio"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x0a" function="0x0"/>
    </disk>
    <disk type="file" device="cdrom">
      <driver name="qemu" type="raw"/>
      <target dev="hda" bus="ide"/>
      <readonly/>
      <address type="drive" controller="0" bus="0" target="0" unit="0"/>
    </disk>
    <controller type="usb" index="0" model="ich9-ehci1">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x05" function="0x7"/>
    </controller>
    <controller type="usb" index="0" model="ich9-uhci1">
      <master startport="0"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x05" function="0x0" multifunction="on"/>
    </controller>
    <controller type="usb" index="0" model="ich9-uhci2">
      <master startport="2"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x05" function="0x1"/>
    </controller>
    <controller type="usb" index="0" model="ich9-uhci3">
      <master startport="4"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x05" function="0x2"/>
    </controller>
    <controller type="pci" index="0" model="pci-root"/>
    <controller type="ide" index="0">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x1"/>
    </controller>
    <controller type="virtio-serial" index="0">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x06" function="0x0"/>
    </controller>
    <interface type="bridge">
      <mac address="52:54:00:83:c1:a8"/>
      <source bridge="br0"/>
      <model type="virtio"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x03" function="0x0"/>
    </interface>
    <serial type="pty">
      <target type="isa-serial" port="0">
        <model name="isa-serial"/>
      </target>
    </serial>
    <console type="pty">
      <target type="serial" port="0"/>
    </console>
    <channel type="unix">
      <target type="virtio" name="org.qemu.guest_agent.0"/>
      <address type="virtio-serial" controller="0" bus="0" port="1"/>
    </channel>
    <channel type="spicevmc">
      <target type="virtio" name="com.redhat.spice.0"/>
      <address type="virtio-serial" controller="0" bus="0" port="2"/>
    </channel>
    <input type="tablet" bus="usb">
      <address type="usb" bus="0" port="1"/>
    </input>
    <input type="mouse" bus="ps2"/>
    <input type="keyboard" bus="ps2"/>
    <graphics type="spice" autoport="yes">
      <listen type="address"/>
    </graphics>
    <video>
      <model type="qxl" ram="65536" vram="65536" vgamem="16384" heads="1" primary="yes"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0"/>
    </video>
    <redirdev bus="usb" type="spicevmc">
      <address type="usb" bus="0" port="2"/>
    </redirdev>
    <redirdev bus="usb" type="spicevmc">
      <address type="usb" bus="0" port="3"/>
    </redirdev>
    <memballoon model="virtio">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x08" function="0x0"/>
    </memballoon>
  </devices>
</domain>

What is wrong ? How can I fix it ?

3
  • Something is very wrong with the free -m output from your guest. The buff/cache number is much higher than it could possibly actually be, given the rest of the numbers. Have you changed this VM's memory configuration while it is running? Aug 19, 2020 at 6:56
  • No it is set as it since a while. A few Host reboots occurred following kernel upgrade.
    – Laurent
    Aug 19, 2020 at 7:00
  • So I have stopped the VM. 32GB of memory was still used by ... well I don't know (ps_mem wasn't showing that much memory uised). So I rebooted the host. And the problem is back after a while ...
    – Laurent
    Aug 20, 2020 at 0:18

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy