NVIDIA GPU 사용률 기록 (1초 마다)

 

# vi gpu_logger.sh

#!/bin/bash

LOG_FILE="gpu_usage_$(date +%Y%m%d_%H%M%S).log"

echo "timestamp,avg_utilization_gpu(%),avg_utilization_mem(%),avg_memory_used(MiB),avg_memory_total(MiB)" >> $LOG_FILE

while true
do
  TIMESTAMP=$(date "+%Y-%m-%d %H:%M:%S")

  # GPU 정보를 CSV 형식으로 가져오기 (이름은 제외)
  GPU_DATA=$(nvidia-smi --query-gpu=utilization.gpu,utilization.memory,memory.used,memory.total --format=csv,noheader,nounits)

  # awk로 각 열을 집계하여 평균 계산
  AGGREGATED=$(echo "$GPU_DATA" | awk -F ', *' '{
      sum1 += $1; sum2 += $2; sum3 += $3; sum4 += $4; count++
    } END {
      if(count > 0)
        printf("%.2f,%.2f,%.2f,%.2f", sum1/count, sum2/count, sum3/count, sum4/count);
    }')

  echo "$TIMESTAMP,$AGGREGATED" >> $LOG_FILE
  sleep 1
done