用 process_vm_readv() 实现多进程内存队列

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192

#include <assert.h>
#include <fcntl.h>
#include <mqueue.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/uio.h>
#include <sys/wait.h>
#include <unistd.h>

#include <iostream>
#include <sstream>
#include <vector>
using namespace std;

struct MemMessage {
    pid_t pid = 0;
    vector<iovec> mem_vec;

public:
    MemMessage() {}

    string DebugString() const {
        ostringstream oss;
        oss << "pid=" << pid;
        for (const auto& mem : mem_vec) {
            oss << " iov={" << mem.iov_base << "," << mem.iov_len << "}";
        }
        return oss.str();
    }

    string ToString() const {
        pid_t from_pid = getpid();
        string buff;
        buff.append((const char*)&from_pid, sizeof(from_pid));
        for (const auto& mem : mem_vec) {
            buff.append((const char*)&mem, sizeof(mem));
        }
        return buff;
    }

    bool FromString(const char* buff, size_t buff_len) {
        size_t pos = 0;
        if (buff_len < sizeof(pid)) {
            return false;
        }
        pid = *((pid_t*)buff);
        pos += sizeof(pid);
        for (; pos + sizeof(iovec) <= buff_len; pos += sizeof(iovec)) {
            mem_vec.resize(mem_vec.size() + 1);
            auto& iov = mem_vec.back();
            memcpy(reinterpret_cast<char*>(&iov), buff + pos, sizeof(iov));
        }
        return true;
    }

    void CopyToMem(string& buff) {
        size_t sum = 0;
        for (const auto& iov : mem_vec) {
            sum += iov.iov_len;
        }

        buff.resize(sum, '\0');
        iovec local;
        local.iov_base = &buff[0];
        local.iov_len = buff.size();

        auto nread = process_vm_readv(pid, &local, 1, &mem_vec[0], mem_vec.size(), 0);
        // assert(nread >= 0);
        if (nread <= 0) {
            cout << getpid() << " process_vm_readv from=" << DebugString() << " sum=" << sum << " len=" << nread
                 << endl;
            cout << "pid=" << pid << " " << strerror(errno) << endl;
        }
    }
};

class MemQueue {
    mqd_t m_mq;
    struct mq_attr m_attr;

public:
    int Init(const string& queue_name, bool read_write = false) {
        memset(&m_attr, 0, sizeof(m_attr));
        m_attr.mq_flags = 0;
        m_attr.mq_maxmsg = 10;
        m_attr.mq_msgsize = 4096;
        const int oflag = read_write ? O_WRONLY | O_CREAT : O_RDONLY;
        m_mq = mq_open(queue_name.c_str(), oflag, S_IRWXU, &m_attr);
        // m_mq = mq_open(queue_name.c_str(), oflag);
        cout << getpid() << " mq_open=" << queue_name << " oflag=" << oflag << " ret=" << m_mq << endl;
    }

    int PushMemMsg(const MemMessage& msg) {
        string buff = msg.ToString();
        return mq_send(m_mq, buff.c_str(), buff.size(), 0);
    }

    bool PopMemMsg(MemMessage& msg) {
        string buff(m_attr.mq_msgsize, '\0');
        unsigned prio = 0;
        const auto ret = mq_receive(m_mq, &buff[0], buff.size(), &prio);
        if (ret < 0) {
            return false;
        }
        buff.resize(ret);
        msg.FromString(buff.data(), buff.size());
        return true;
    }
};

const static string queue_name = "/test_queue";

enum STATUS { STATUS_WAITING = 0, STATUS_RUNNING = 1, STATUS_EXIT = 2 };

struct ControlCenter {
    int proc_status;
};

ControlCenter* control = nullptr;

void child_proc() {
    const auto ret = fork();
    assert(ret >= 0);
    if (ret != 0) {
        return;
    }

    while (STATUS_WAITING == control->proc_status) {
        usleep(1000);
    }

    MemQueue q;
    q.Init(queue_name);
    size_t bytes = 0;
    while (STATUS_RUNNING == control->proc_status) {
        MemMessage msg;
        if (q.PopMemMsg(msg)) {
            string buff;
            msg.CopyToMem(buff);
            bytes += buff.size();
        } else {
            usleep(1000);
        }
    }
    cout << getpid() << " bytes=" << bytes << " exit" << endl;

    exit(0);
}

int main(int argc, char* argv[]) {
    control = (ControlCenter*)mmap(NULL, sizeof(*control), PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
    control->proc_status = STATUS_WAITING;

    const static int proc_num = 3;
    for (int i = 0; i < proc_num; ++i) {
        child_proc();
    }

    MemQueue q;
    q.Init(queue_name, true);

    control->proc_status = STATUS_RUNNING;
    string str(1024 * 1024 * 10, 'a');
    for (int i = 0; i < 10000; ++i) {
        MemMessage msg;
        for (int t = 0; t < 3; ++t) {
            size_t base = rand() % str.size();
            size_t size = rand() % (str.size() - base);
            msg.mem_vec.emplace_back(iovec{&str[base], size});
        }
        const auto ret = q.PushMemMsg(msg);
        // cout << getpid() << " PushMemMsg=" << msg.DebugString() << " ret=" << ret << endl;

        // sleep(1);
    }

    control->proc_status = STATUS_EXIT;

    for (int i = 0; i < proc_num; ++i) {
        int status = 0;
        wait(&status);
    }

    return 0;
}