다중 Ollama Load Balancer (NGINX 설정)

아래는 ollama1, ollama2 서버에 각각 GPU 두개씩가 설치되어 NUM_PARALLEL=6 으로 ollama server 중인 상태에서,

요청을 적절히 분기해주기 위한 reverse proxy 설정 예시입니다.

dify plugin 으로 ollama 호출 시, .env 의 CELERY_WORKER_AMOUNT도 적절히 늘려야 합니다. (ex: 6*4=24)

안그러면 default 값인 1로 인해 한번에 1개씩만 호출되어 LB의 의미가 없습니다.

user www-data;
worker_processes auto;
pid /run/nginx.pid;
error_log /var/log/nginx/error.log warn;

include /etc/nginx/modules-enabled/*.conf;

events {
    worker_connections 2048;
    multi_accept on;
}

http {
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    types_hash_max_size 2048;
    server_tokens off;
    include /etc/nginx/mime.types;
    default_type application/octet-stream;
    gzip on;
    gzip_types text/plain application/json;

    log_format upstream_log '$remote_addr - $remote_user [$time_local] '
                            '"$request" $status $body_bytes_sent '
                            'upstream: $upstream_addr';
    access_log /var/log/nginx/access.log upstream_log;

    upstream ollama_cluster {
        least_conn;
        server ollama1:11434 max_conns=6 max_fails=1 fail_timeout=1s;
        server ollama1:11435 max_conns=6 max_fails=1 fail_timeout=1s;
        server ollama2:11434 max_conns=6 max_fails=1 fail_timeout=1s;
        server ollama2:11435 max_conns=6 max_fails=1 fail_timeout=1s;
        keepalive 64;
    }

    server {
        listen 11433;
        server_name _;
        proxy_read_timeout 3600s;
        proxy_connect_timeout 3600s;
        proxy_send_timeout 3600s;
        client_max_body_size 100M;
        proxy_buffering off;
        proxy_request_buffering off;
        proxy_cache off;

        location / {
            proxy_pass http://ollama_cluster;
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header Keep-Alive "";
            keepalive_timeout 0;

            chunked_transfer_encoding on;
        }
    }
}