SgtSmuckers/ProxMon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: SgtSmuckers:34b7048d2f80f05150fb8317588ba58d9a3beffb
Branches Tags
SgtSmuckers:main
..
compare: SgtSmuckers:e899763d66a22d9491094d84fdb72a0e5e786308
Branches Tags
SgtSmuckers:main

No commits in common. "34b7048d2f80f05150fb8317588ba58d9a3beffb" and "e899763d66a22d9491094d84fdb72a0e5e786308" have entirely different histories.
34b7048d2f ... e899763d66

14 changed files with 1642 additions and 1146 deletions
Show Stats Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,5 +1,4 @@
# Prerequisites
*.service
*.d
# Object files

70
Makefile
View File

@ -15,76 +15,22 @@ INC_FLAGS := $(addprefix -I,$(INC_DIRS))
CCFLAGS := $(INC_FLAGS)
.PHONY: build clean install install-services uninstall-services create-services reinstall-services
install: build create-services install-services ## Full installation. Builds the proxmon executable, generates service files, installs and starts services
build: $(TARGET) create-services ## Build the proxmon executable, and generates service files for editing
clean: ## Remove all build artifacts including generated service files (does not uninstall service files)
@rm -rf bin/ $(OUT_DIR) $(SERVICE_DIR)
./bin:
@mkdir -p ./bin
@cp ./dist/* ./bin
$(OUT_DIR)/%.o: $(SRC_DIRS)/%.c
@mkdir -p $(dir $@)
$(OUT_DIR):
@mkdir -p $@
$(OUT_DIR)/%.o: $(SRC_DIRS)/%.c | $(OUT_DIR)
$(CC) $(CCFLAGS) -c $< -o $@
$(TARGET): $(OBJS) | ./bin
$(CC) $(CCFLAGS) $(OBJS) -o $@
.PHONY: all clean
all: $(TARGET)
TEMPLATE_DIR ?= ./template
SERVICE_DIR ?= ./systemd
INSTALL_DIR ?= /etc/systemd/system
WORKDIR := $(shell pwd)
clean:
@rm -rf bin/ $(OUT_DIR)
# Collect all template service files
TEMPLATES := $(shell find $(TEMPLATE_DIR) -name '*.service.in')
# Generate names of service files that will have paths substituted
SERVICES := $(patsubst $(TEMPLATE_DIR)/%.service.in, $(SERVICE_DIR)/%.service, $(TEMPLATES))
# Generate names of installed service files
INSTALLED := $(patsubst $(SERVICE_DIR)/%, $(INSTALL_DIR)/%, $(SERVICES))
# Create services directory
$(SERVICE_DIR):
@mkdir -p $@
# Convert template service into actual service file
$(SERVICE_DIR)/%.service: $(TEMPLATE_DIR)/%.service.in | $(SERVICE_DIR)
sed 's|@WORKDIR@|$(WORKDIR)|g' $< > $@
create-services: $(SERVICES) ## Generate service files from templates into the systemd/ directory
# install a service into systemd
$(INSTALL_DIR)/%.service: $(SERVICE_DIR)/%.service
install -m 644 $< $(INSTALL_DIR)/
install-services: create-services $(INSTALLED) ## Installs all service files in the systemd/ directory into /etc/systemd/system
systemctl daemon-reload
@for srv in $(SERVICES); do \
echo "Installing and starting service $$srv..."; \
name=$$(basename $$srv); \
systemctl enable $$name; \
systemctl start $$name; \
done
uninstall-services: ## Stops and uninstalls all services associated with ProxMon
@for srv in $(INSTALLED); do \
name=$$(basename $$srv); \
echo "Stopping and uninstalling $$name..."; \
systemctl stop $$name || true; \
systemctl disable $$name || true; \
if [ -f $$srv ]; then rm -f $$srv; fi; \
done
systemctl daemon-reload
reinstall-services: uninstall-services install-services ## Uninstalls and reinstalls ProxMon services
help: ## Show help message
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[$$()% a-zA-Z_-]+:.*?##/ { printf " \033[36m%-18s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

1576
Proxmox-Dashboard.json → Prometheus Node Exporter Full.json
View File

File diff suppressed because it is too large Load Diff

81
README.md
View File

@ -1,75 +1,16 @@
# ProxMon
Basic idea:
Use the following PS command to gather basic data on processes:
ps ax -o pid,ppid,tgid,uid,lxc,cuu,rss,times,etimes,stat,command --cols 1000
Use resident size as an approximation of real memory usage.
Prometheus exporter that monitors LXC, process and node metrics for Proxmox. This is comprised of two components:
1. node-exporter
- Publicly available "node exporter" from Prometheus. This provides a wide range of system metrics, and by default serves metrics on port 9100.
- Pre-built binary in this repo for ease of installation.
- Configuration details can be found [here](https://github.com/prometheus/node_exporter).
2. proxmon
- Provides supplemental metrics that node-exporter does not provide, and by default serves metrics on port 9101.
- Metrics include resource usage on a per-LXC and per-process basis.
Use CPU utilization may need to be divided by CPU count
Also included is a Grafana dashboard tailored to the data exported by both components above.
# Installation
1. Install dependencies needed to build
```apt install git make gcc```
2. Create a directory for ProxMon to live
```mkdir /home/monitor```
**NOTE**: This does not create a new user. This guide does not include configuring the service to run under a different user, and will by default run it under root. See **Customizing Services**.
3. ```cd /home/monitor```
4. ```git clone https://git.cozyclan.xyz/dan/ProxMon.git```
5. ```cd ProxMon```
6. If you wish to configure the services in any way, such as running them as a new user (note on step 2) or editing the port they use, skip the remaining steps and go to the **Customizing Services** section.
7. Run ```make install```
8. Verify services are running:
```systemctl status node-exporter.service```
```systemctl status proxmon-exporter.service```
9. Congratulations, you're done. Go to the **Connecting** section.
## Customizing Services
The Makefile has various build targets that can be used for finer control over the installation process. A full list of targets can be viewed with ```make help```.
1. Build proxmon: ```make build```. This will place the executables in the ```bin/``` folder, and generate template service files in the ```systemd/``` directory.
2. Edit the service files in the ```systemd/``` directory to enable node-exporter features or change the ports of the components. If you would like to run these as non-root users, [ask ChatGPT](https://chatgpt.com/share/6830a1d9-5fe0-8012-abdd-076e78a12067).
3. Install the service files: ```make install-services```. This will copy the service files from ```systemd/``` into ```/etc/systemd/system``` and will start the services in systemd. You can also use ```make reinstall-services``` if you have already installed the services and want to install updated versions of the service files.
## Running inside an LXC
This is intended to be run on the host system. However, it is probably possible to run it in an LXC. Setting up a privileged LXC would probably be the most straightforward, but an unprivileged should be possible. It would just be annoying to set up because you'd have to map UIDs/GIDs between the LXC and host.
Either way, you will need to bind-mount the hosts root directory into the LXC, and point both node-exporter and proxmon components to read from the bind-mounted directory instead of the default ```/proc``` directory of the LXC.
If for instance you bind-mounted the hosts root directory to ```/host``` in the LXC, you could add the startup parameter to node-exporter: ```--path.rootfs=/host```
And the following startup parameter to proxmon: ```--path.rootfs /host```. See the **Customizing Services** section.
# Connecting
## Prometheus
0. Install Prometheus if you haven't already with the [Prometheus Helper Script](https://community-scripts.github.io/ProxmoxVE/scripts?id=prometheus). Save the web URL displayed at the end of the setup script.
1. Open your Prometheus LXC console.
2. Edit the file ```/etc/prometheus/prometheus.yml```
3. Under ```scrape_configs``` add the following job. You can edit the job_name as you wish. Set the IP address in the targets to the IP of the host you wish to scrape data from.
4. You can also edit the polling rates in the yml file.
```
scrape_configs:
- job_name: "node"
static_configs:
- targets: ['<HOST_IP>:9100', '<HOST_IP>:9101']
```
5. Restart prometheus: ```systemctl restart prometheus.service```
## Grafana
0. Install Grafana if you haven't already with the [Grafana Helper Script](https://community-scripts.github.io/ProxmoxVE/scripts?id=grafana). Save the web URL displayed at the end of the setup script.
1. Login to the Grafana web interface provided at the end of the helper script
2. On the left side, go to ```Connections->Add new connection```.
3. Search for and click on "Prometheus".
4. Click ```Add new data source``` in the top right.
5. Under ```Connection``` and ```Prometheus server URL``` paste in the web URL of your Prometheus server listed at the end of the Prometheus Helper Script.
6. On the left side, go to ```Dashboards```
7. In the top right, click ```New->Import```
8. Copy the contents of ```Proxmox-Dashboard.json``` from this git repository, and paste it into the text field.
9. Click ```Import```
Structure of each entry:
_id: unique ID of data point
time: time sample was taken
node: Node of process
host: Host name
hostType: Type of host ('node', 'lxc', 'vm')

76
py/ps_fields.py Normal file
View File

@ -0,0 +1,76 @@
field_defs = {
'PID': {
'desc': 'PID',
'dtype': str,
},
'PPID': {
'desc': 'Parent PID',
'dtype': str,
},
'UID': {
'desc': 'User ID',
'dtype': str,
},
'LXC': {
'desc': 'LXC ID',
'dtype': str,
},
'%CUU': {
'desc': 'CPU Use',
'dtype': float,
},
'%MEM': {
'desc': 'Memory Use',
'dtype': float,
},
'VSZ': {
'desc': 'Virtual Memory',
'dtype': int,
'mult': 1024,
},
'RSS': {
'desc': 'Resident Memory',
'dtype': int,
'mult': 1024,
},
'PSS': {
'desc': 'Proportional Shared Memory',
'dtype': int,
'mult': 1024,
},
'USS': {
'desc': 'Unique Memory',
'dtype': int,
'mult': 1024,
},
'SZ': {
'desc': 'Core Size',
'dtype': int,
'mult': 4096,
},
'SIZE': {
'desc': 'Approximate Memory',
'dtype': int,
'mult': 1024,
},
'THCNT': {
'desc': 'Thread Count',
'dtype': int,
},
'TIME': {
'desc': 'Execution Time',
'dtype': int,
},
'ELAPSED': {
'desc': 'Elapsed Time',
'dtype': int,
},
'STAT': {
'desc': 'Status',
'dtype': str,
},
'COMMAND': {
'desc': 'Command',
'dtype': str,
}
}

33
py/ps_parse.py Normal file
View File

@ -0,0 +1,33 @@
from ps_fields import *
import mongodb
def parsePsOutput(output):
data = output.split('\n')
data = list(filter(bool, [line.split() for line in data]))
header = data[0]
procs = []
for line in data[1:]:
proc = {}
for i in range(len(header)):
field = field_defs[header[i]]
value = line[i]
if i == len(header):
# combine command arguments together to one string
value = ' '.join(line[i:])
value = field['dtype'](value)
if 'mult' in field:
# mult if applicable
value *= field['mult']
proc[header[i]] = value
procs.append(proc)
return procs
def parseFile(fname):
f = open(fname, 'r')
data = f.read()
f.close()
return parsePsOutput(data)

39
py/stat_flags.py Normal file
View File

@ -0,0 +1,39 @@
reverse_lookup = {
0x00000001: "PF_VCPU", # I'm a virtual CPU
0x00000002: "PF_IDLE", # I am an IDLE thread
0x00000004: "PF_EXITING", # Getting shut down
0x00000008: "PF_POSTCOREDUMP", # Coredumps should ignore this task
0x00000010: "PF_IO_WORKER", # Task is an IO worker
0x00000020: "PF_WQ_WORKER", # I'm a workqueue worker
0x00000040: "PF_FORKNOEXEC", # Forked but didn't exec
0x00000080: "PF_MCE_PROCESS", # Process policy on mce errors
0x00000100: "PF_SUPERPRIV", # Used super-user privileges
0x00000200: "PF_DUMPCORE", # Dumped core
0x00000400: "PF_SIGNALED", # Killed by a signal
0x00000800: "PF_MEMALLOC", # Allocating memory
0x00001000: "PF_NPROC_EXCEEDED", # set_user() noticed that RLIMIT_NPROC was exceeded
0x00002000: "PF_USED_MATH", # If unset the fpu must be initialized before use
0x00004000: "PF_USER_WORKER", # Kernel thread cloned from userspace thread
0x00008000: "PF_NOFREEZE", # This thread should not be frozen
0x00010000: "PF__HOLE__00010000",
0x00020000: "PF_KSWAPD", # I am kswapd
0x00040000: "PF_MEMALLOC_NOFS", # All allocation requests will inherit GFP_NOFS
0x00080000: "PF_MEMALLOC_NOIO", # All allocation requests will inherit GFP_NOIO
0x00100000: "PF_LOCAL_THROTTLE", # Throttle writes only against the bdi I write to, I am cleaning dirty pages from some other bdi.
0x00200000: "PF_KTHREAD", # I am a kernel thread
0x00400000: "PF_RANDOMIZE", # Randomize virtual address space
0x00800000: "PF__HOLE__00800000",
0x01000000: "PF__HOLE__01000000",
0x02000000: "PF__HOLE__02000000",
0x04000000: "PF_NO_SETAFFINITY", # Userland is not allowed to meddle with cpus_mask
0x08000000: "PF_MCE_EARLY", # Early kill for mce process policy
0x10000000: "PF_MEMALLOC_PIN", # Allocation context constrained to zones which allow long term pinning.
0x20000000: "PF__HOLE__20000000",
0x40000000: "PF__HOLE__40000000",
0x80000000: "PF_SUSPEND_TASK", # This thread called freeze_processes() and should not be frozen
}
def listFlags(flag):
for i in reverse_lookup.keys():
if flag & i:
print(reverse_lookup[i])

25
src/lxcstat.c
View File

@ -24,19 +24,16 @@ int parseLxcConfigFile(lxcinfo *lxc, char *filedata) {
strcpy(lxc->hostname, ptr);
}
int readLxcConfig(lxcinfo *lxc, int lxcid, char *pathPrefix) {
int readLxcConfig(lxcinfo *lxc, int lxcid) {
char *fname = malloc(1024);
char *buffer = malloc(4096);
sprintf(fname, "%s/etc/pve/lxc/%d.conf", pathPrefix, lxcid);
sprintf(fname, "/etc/pve/lxc/%d.conf", lxcid);
FILE *file = fopen(fname, "rb");
if (file == NULL) {
fprintf(stderr, "Failed to open <%s>: %d\n", fname, errno);
free(buffer);
free(fname);
printf("Failed to open <%s>: %d\n", fname, errno);
return -1;
}
fread(buffer, 1, 4096, file);
fclose(file);
@ -46,17 +43,14 @@ int readLxcConfig(lxcinfo *lxc, int lxcid, char *pathPrefix) {
return 0;
}
void getInactiveLXCs(lxcinfo **lxcs, char *pathPrefix) {
void getInactiveLXCs(lxcinfo **lxcs) {
struct dirent *pDirent;
DIR *pDir;
char *fname = malloc(1024);
sprintf(fname, "%s/etc/pve/lxc", pathPrefix);
// Ensure we can open directory.
pDir = opendir(fname);
pDir = opendir("/etc/pve/lxc");
if (pDir == NULL) {
fprintf(stderr, "Cannot open directory '%s'\n", fname);
free(fname);
printf("Cannot open directory '/etc/pve/lxc'\n");
return;
}
@ -69,14 +63,13 @@ void getInactiveLXCs(lxcinfo **lxcs, char *pathPrefix) {
int argc = sscanf(fname, "%d.%s", &lxcid, (char *) extension);
// ensure it's an LXC config
if (argc != 2 || strcmp((char *) extension, "conf") != 0) continue;
getLXCInfo(lxcs, lxcid, pathPrefix); // don't care about result
getLXCInfo(lxcs, lxcid); // don't care about result
}
closedir(pDir);
free(fname);
}
lxcinfo *getLXCInfo(lxcinfo **lxcs, int lxcid, char *pathPrefix) {
lxcinfo *getLXCInfo(lxcinfo **lxcs, int lxcid) {
lxcinfo *lxc = *lxcs;
while (lxc != NULL) {
if (lxc->lxcid == lxcid) return lxc;
@ -87,7 +80,7 @@ lxcinfo *getLXCInfo(lxcinfo **lxcs, int lxcid, char *pathPrefix) {
lxc->lxcid = lxcid;
lxc->running = 0;
if (readLxcConfig(lxc, lxcid, pathPrefix)) {
if (readLxcConfig(lxc, lxcid)) {
// failed to read LXC
free(lxc);
return NULL;

4
src/lxcstat.h
View File

@ -13,7 +13,7 @@ typedef struct _lxc_info {
char hostname[128];
} lxcinfo;
lxcinfo *getLXCInfo(lxcinfo **lxcs, int lxcid, char *pathPrefix);
void getInactiveLXCs(lxcinfo **lxcs, char *pathPrefix);
lxcinfo *getLXCInfo(lxcinfo **lxcs, int lxcid);
void getInactiveLXCs(lxcinfo **lxcs);
#endif

19
src/process.c
View File

@ -240,19 +240,16 @@ nextProcess:
}
}
char *readProcesses(char *pathPrefix) {
char *readProcesses(char *procdir) {
int len;
struct dirent *pDirent;
DIR *pDir;
char *procdir = malloc(1024);
sprintf(procdir, "%s/proc", pathPrefix);
// Ensure we can open directory.
pDir = opendir(procdir);
if (pDir == NULL) {
printf("Cannot open directory '%s'\n", procdir);
free(procdir);
return NULL;
}
@ -275,7 +272,6 @@ char *readProcesses(char *pathPrefix) {
while ((pDirent = readdir(pDir)) != NULL) {
first = pDirent->d_name[0];
// Ignore directories that aren't PIDs
if (first < '0' || first > '9') continue;
if (cur->visited) {
@ -285,7 +281,7 @@ char *readProcesses(char *pathPrefix) {
cur = cur->next;
}
sprintf(fname, "%s/%s/status", procdir, pDirent->d_name);
sprintf(fname, "/proc/%s/status", pDirent->d_name);
file = fopen(fname, "rb");
if (file == NULL) continue;
fread(buffer, 1, 4096, file);
@ -303,7 +299,7 @@ char *readProcesses(char *pathPrefix) {
if (parseStatFile(cur, buffer)) continue;
sprintf(fname, "%s/%s/io", procdir, pDirent->d_name);
sprintf(fname, "/proc/%s/io", pDirent->d_name);
file = fopen(fname, "rb");
if (file == NULL) continue;
fread(buffer, 1, 4096, file);
@ -311,7 +307,7 @@ char *readProcesses(char *pathPrefix) {
if (parseIOFile(cur, buffer)) continue;
sprintf(fname, "%s/%s/cpuset", procdir, pDirent->d_name);
sprintf(fname, "/proc/%s/cpuset", pDirent->d_name);
file = fopen(fname, "rb");
if (file == NULL) goto nextProcess;
fread(cur->cpuset, 1, sizeof(cur->cpuset), file);
@ -322,7 +318,7 @@ char *readProcesses(char *pathPrefix) {
if (memcmp(cur->cpuset, lxcTag, strlen(lxcTag)) == 0) {
// Resides in LXC -- read file and tag
sscanf(cur->cpuset, "/lxc/%d/ns", &cur->lxc);
lxcinfo *lxc = getLXCInfo(&lxcs, cur->lxc, pathPrefix);
lxcinfo *lxc = getLXCInfo(&lxcs, cur->lxc);
if (lxc == NULL) {
printf("Failed to read LXC config <%d>\n", cur->lxc);
} else {
@ -344,7 +340,7 @@ nextProcess:
linkFamily(head);
aggregateStats(head);
getInactiveLXCs(&lxcs, pathPrefix);
getInactiveLXCs(&lxcs);
int clocks = sysconf(_SC_CLK_TCK);
char *output = malloc(8 * 1024 * 1024);
@ -394,7 +390,6 @@ nextProcess:
}
closedir(pDir);
free(procdir);
free(buffer);
free(fname);
return output;

2
src/process.h
View File

@ -35,7 +35,7 @@ typedef struct st_process {
uint64_t cWrite;
} Process;
char *readProcesses(char *pathPrefix);
char *readProcesses(char *procdir);
void aggregateStats(Process *head);
int resetVisits(Process *head);

36
src/server.c
View File

@ -44,8 +44,8 @@ void *self_connector_thread(void *arg) {
#define BUFFER_SIZE 1024
void handle_client(int client_socket, char *pathPrefix) {
char *data = readProcesses(pathPrefix);
void handle_client(int client_socket) {
char *data = readProcesses("/proc");
int length = strlen(data);
printf("Got data of length: %d\n", length);
@ -92,32 +92,22 @@ closeConnection:
}
int main(int argc, char *argv[]) {
char *portStr = NULL;
char *pathPrefix = "";
for (int i=0; i<argc; i++) {
if (strcmp("--port", argv[i]) == 0) {
portStr = argv[++i];
if (argc == 1) {
char *buf = readProcesses("/proc");
if (buf == NULL) return 1;
printf(buf);
free(buf);
return 0;
}
if (strcmp("--path.rootfs", argv[i]) == 0) {
pathPrefix = argv[++i];
}
}
if (portStr == NULL) {
fprintf(stderr, "Missing required argument --port\n");
return 1;
}
for (int i=0; i<strlen(portStr); i++) {
if (portStr[i] < '0' || portStr[i] > '9') {
fprintf(stderr, "Argument '%s' is not a valid port\n", portStr);
for (int i=0; i<strlen(argv[1]); i++) {
if (argv[1][i] < '0' || argv[1][i] > '9') {
fprintf(stderr, "Argument '%s' is not a valid port", argv[1]);
return 1;
}
}
int port = atoi(portStr);
int port = atoi(argv[1]);
// Set up signal handling
struct sigaction sa;
sa.sa_handler = handle_signal;
@ -176,7 +166,7 @@ int main(int argc, char *argv[]) {
}
// Handle the client in a separate function
handle_client(client_socket, pathPrefix);
handle_client(client_socket);
}
close(client_socket);

15
template/node-exporter.service.in
View File

@ -1,15 +0,0 @@
[Unit]
Description=Prometheus Node Exporter
# Adjust based on dependencies, if any
After=network.target
[Service]
ExecStart=@WORKDIR@/bin/node_exporter --collector.mountstats --no-collector.processes --no-collector.arp --no-collector.nfs --no-collector.nfsd --no-collector.thermal_zone --web.listen-address=:9100
# Restart if the process crashes
Restart=always
# Wait 5 seconds before restarting
RestartSec=5
WorkingDirectory=@WORKDIR@
[Install]
WantedBy=multi-user.target

15
template/proxmon-exporter.service.in
View File

@ -1,15 +0,0 @@
[Unit]
Description=Proxmox Process Monitor/Exporter
# Adjust based on dependencies, if any
After=network.target
[Service]
ExecStart=@WORKDIR@/bin/proxmon --port 9101
# Restart if the process crashes
Restart=always
# Wait 5 seconds before restarting
RestartSec=5
WorkingDirectory=@WORKDIR@
[Install]
WantedBy=multi-user.target