First of all, I've searched in stackoverflow.com and googled, but no effective results are found.

My question is: Why does pthread_cond_wait consume so much CPU? I don't think it's normal.

My program suffered that %CPU became high intermittently and stayed high for more than ten seconds. When %CPU was stably low, it was around 1. When it became high, it was between 50 and 300.

I used top -H -p to find the single thread consuming the most CPU when %CPU of the process became high, and then I used strace -T -r -c -p to find more information:

strace -T -r -c -p 1701

% time     seconds  usecs/call     calls    errors syscall
------ ----------- ----------- --------- --------- ----------------
 88.54    0.482646          43     11157      3020 futex
  9.85    0.053682           0    131052           read
  1.50    0.008192          38       213           nanosleep
  0.04    0.000214           1       239           write
  0.03    0.000154           1       213           open
  0.02    0.000111           1       213           munmap
  0.02    0.000085           0       239           stat
  0.01    0.000044           0       213           mmap
  0.00    0.000018           0       213           close
  0.00    0.000000           0       213           fstat
  0.00    0.000000           0       213           lseek
------ ----------- ----------- --------- --------- ----------------
100.00    0.545146                144178      3020 total

And the stack of that thread:

Thread 6 (Thread 0x7f1404f41700 (LWP 1701)):

#0  0x0000003d6f60b63c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x0000000000406045 in foo(void*) ()
#2  0x0000003d6f607a51 in start_thread () from /lib64/libpthread.so.0
#3  0x0000003d6eee893d in clone () from /lib64/libc.so.6

And related code snippet:

static std::deque<std::string> conveyor;
static pthread_mutex_t conveyor_mtx = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t conveyor_cond = PTHREAD_COND_INITIALIZER;

#define POP_NUM 4

static void *foo(void *arg)
{
    write_log(LOG_INFO, "thread foo created");

    int ret = pthread_detach(pthread_self());
    if (ret != 0) {
        write_log(LOG_ERR, "pthread_detach[foo] failed with errno[%d]", ret);
        write_log(LOG_ERR, "thread foo exiting");
        return (void *)-1;
    }

    std::string paths[POP_NUM], topic;
    int n;

    do {
        if ((ret = pthread_mutex_lock(&conveyor_mtx)) != 0) {
            write_log(LOG_WARNING, "pthread_mutex_lock[conveyor_mtx] failed"
                      " with errno[%d]", ret);
            sleep(1);
            continue;
        }
        while (conveyor.empty()) {
            write_log(LOG_INFO, "conveyor empty");
            pthread_cond_wait(&conveyor_cond, &conveyor_mtx);
        }
        for (n = 0; n < POP_NUM; n++) {
            paths[n].assign(conveyor.front());
            conveyor.pop_front();
            if (conveyor.empty()) break;
        }
        if ((ret = pthread_mutex_unlock(&conveyor_mtx)) != 0) {
            write_log(LOG_WARNING, "pthread_mutex_unlock[conveyor_mtx] failed"
                      " with errno[%d]", ret);
        }
        for (int i = 0; i < n; i++) {
            if (!extract_topic_from_path(paths[i], topic)) continue;
            produce_msgs_and_save_offset(topics[topic],
                                      const_cast<char *>(paths[i].c_str()));
        }
    } while (true);

    write_log(LOG_ERR, "thread foo exiting");

    return (void *)0;
}

static void *bar(void *arg)
{
    write_log(LOG_INFO, "thread bar created");

    int inot_fd = (int)(intptr_t)arg, n, ret;
    struct pollfd pfd = { inot_fd, POLLIN | POLLPRI, 0 };

    do {
        //n = poll(&pfd, 1, -1);
        //n = poll(&pfd, 1, 300000);
        n = poll(&pfd, 1, 120000);
        if (n == -1) {
            if (errno == EINTR) {
                write_log(LOG_WARNING, "poll interrupted by a signal");
                continue;
            }
            write_log(LOG_ERR, "poll failed with errno[%d]", errno);
            write_log(LOG_ERR, "thread bar exiting");
            return (void *)-1;
        } else if (n == 0) {
            write_log(LOG_WARNING, "poll timed out after 120 seconds");
            sleep(60);
        }

        int i;
        for (i = 0; i < 3; i++) {
            if ((ret = pthread_mutex_lock(&conveyor_mtx)) != 0) {
                write_log(LOG_WARNING, "pthread_mutex_lock[conveyor_mtx] failed"
                          "[%d] with errno[%d]", i, ret);
                continue;
            } else {
                break;
            }
        }
        if (i == 3) {
            write_log(LOG_ERR, "thread bar exiting");
            return (void *)-1;
        }
        if ((n = baz(inot_fd)) > 0) {
            pthread_mutex_unlock(&conveyor_mtx);
            pthread_cond_broadcast(&conveyor_cond);
        } else if (n == 0) {
            pthread_mutex_unlock(&conveyor_mtx);
        } else {
            pthread_mutex_unlock(&conveyor_mtx);
            pthread_cond_broadcast(&conveyor_cond);
            write_log(LOG_ERR, "thread bar exiting");
            return (void *)-1;
        }

        if ((n = poll_producer(producer, 1000, 2)) > 0) {
            write_log(LOG_INFO, "rdkafka poll events[%d] of producer"
                      " for possible big outq size", n);
        }
    } while (true);

    write_log(LOG_ERR, "thread bar exiting");

    return (void *)0;
}

And what's more, if I didn't use pthread_cond_wait/pthread_cond_broadcast, and replaced the "pthread_cond_wait" in the above snippet with "sleep", strace would show that the most costly syscall was nanosleep.

uname -a Linux d144122 2.6.32-358.el6.x86_64 #1 SMP Fri Feb 22 00:31:26 UTC 2013 x86_64 x86_64 x86_64 GNU/Linux

I had the same issue. In my case this was caused by the usage of #pragma pack, like in this thread : https://stackoverflow.com/questions/22166474/pthread-cond-wait-doesnt-make-cpu-sleep

Removing the pack solved my problem...

Your Answer

 

By clicking "Post Your Answer", you acknowledge that you have read our updated terms of service, privacy policy and cookie policy, and that your continued use of the website is subject to these policies.

Not the answer you're looking for? Browse other questions tagged or ask your own question.