Master Nginx: Load Balancing, Caching, SSL & Static/Dynamic Separation Guide

Preparation

Use a Debian environment and install Nginx (default installation), a web project, and Tomcat (default installation).

Nginx.conf Configuration

# Define Nginx user and group (use low‑privilege user for security)
# user www www;
# Number of worker processes, usually set to the number of CPU cores
worker_processes 8;
# Global error log level
error_log /var/log/nginx/error.log info;
# PID file
pid /var/run/nginx.pid;
# Maximum number of open file descriptors per worker
worker_rlimit_nofile 65535;

events {
    # Use epoll for high performance
    use epoll;
    # Maximum connections per worker
    worker_connections 65535;
}

http {
    # MIME types mapping
    include mime.types;
    default_type application/octet-stream;
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    # Access and error logs
    access_log /var/log/nginx/access.log;
    error_log /var/log/nginx/error.log;
    # Gzip compression
    gzip on;
    gzip_min_length 1k;
    gzip_buffers 16 64k;
    gzip_http_version 1.1;
    gzip_comp_level 6;
    gzip_types text/plain application/x-javascript text/css application/xml application/javascript;
    gzip_vary on;
    # Load‑balancing groups
    upstream static.zh-jieli.com {
        server 127.0.0.1:808 weight=1;
    }
    upstream zh-jieli.com {
        server 127.0.0.1:8080;
        # server 192.168.8.203:8080;
    }
    # Proxy parameters
    proxy_redirect off;
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    client_max_body_size 10m;
    client_body_buffer_size 128k;
    proxy_connect_timeout 65;
    proxy_send_timeout 65;
    proxy_read_timeout 65;
    proxy_buffer_size 4k;
    proxy_buffers 4 32k;
    proxy_busy_buffers_size 64k;
    # Cache configuration
    proxy_cache_key '$host:$server_port$request_uri';
    proxy_temp_file_write_size 64k;
    proxy_temp_path /dev/shm/JieLiERP/proxy_temp_path;
    proxy_cache_path /dev/shm/JieLiERP/proxy_cache_path levels=1:2 keys_zone=cache_one:200m inactive=5d max_size=1g;
    proxy_ignore_headers X-Accel-Expires Expires Cache-Control Set-Cookie;

    server {
        listen 80;
        server_name erp.zh-jieli.com;
        location / {
            index index;
            #proxy_pass http://jieli;
        }
        location ~ .*(js|css|ico|png|jpg|eot|svg|ttf|woff) {
            proxy_cache cache_one;
            proxy_cache_valid 200 304 302 5d;
            proxy_cache_valid any 5d;
            proxy_cache_key '$host:$server_port$request_uri';
            add_header X-Cache '$upstream_cache_status from $host';
            proxy_pass http://static.zh-jieli.com;
            expires 30d;
        }
        location ~ .*$ {
            index index;
            proxy_pass http://zh-jieli.com;
        }
    }

    server {
        listen 808;
        server_name static;
        location / {
        }
        location ~ .*(js|css|ico|png|jpg|eot|svg|ttf|woff) {
            # Serve static files directly from disk
            root /var/lib/tomcat7/webapps/JieLiERP/WEB-INF;
            expires 30d;
        }
    }
}

The above configuration enables basic load balancing, defines upstream groups for static and dynamic servers, and sets up gzip compression and proxy caching.

Example Explanation

Assume a machine at 192.168.8.203 runs Tomcat with a J2EE service on port 8080. To avoid Tomcat handling static files (which consumes resources), Nginx is used as a reverse proxy to separate static and dynamic content.

worker_processes 8;
pid /var/run/nginx.pid;
worker_rlimit_nofile 65535;

events {
    use epoll;
    worker_connections 65535;
}

http {
    include mime.types;
    default_type application/octet-stream;
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 65;
    types_hash_max_size 2048;
    proxy_redirect off;
    proxy_set_header Host $host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    client_max_body_size 10m;
    client_body_buffer_size 128k;
    proxy_connect_timeout 65;
    proxy_send_timeout 65;
    proxy_read_timeout 65;
    proxy_buffer_size 4k;
    proxy_buffers 4 32k;
    proxy_busy_buffers_size 64k;

    server {
        listen 80;
        server_name xxx.com;
        location / {
            index index;
        }
        location ~ .*(js|css|ico|png|jpg|eot|svg|ttf|woff) {
            proxy_pass http://192.168.8.203:8080;
            expires 30d;
        }
        location ~ .*$ {
            proxy_pass http://192.168.8.203:8080;
        }
    }
}

This simple setup proxies dynamic requests to Tomcat while serving static assets directly.

Gzip Compression

gzip on;
 gzip_min_length 1k;
 gzip_buffers 16 64k;
 gzip_http_version 1.1;
 gzip_comp_level 6;
 gzip_types text/plain application/x-javascript text/css application/xml application/javascript;
 gzip_vary on;

Enabling gzip reduces bandwidth usage for text‑based resources.

Proxy Cache Configuration

# Cache key definition
proxy_cache_key '$host:$server_port$request_uri';
proxy_temp_file_write_size 64k;
proxy_temp_path /dev/shm/JieLiERP/proxy_temp_path;
proxy_cache_path /dev/shm/JieLiERP/proxy_cache_path levels=1:2 keys_zone=cache_one:200m inactive=5d max_size=1g;
proxy_ignore_headers X-Accel-Expires Expires Cache-Control Set-Cookie;

location ~ .*(js|css|ico|png|jpg|eot|svg|ttf|woff) {
    proxy_cache cache_one;
    proxy_cache_valid 200 304 302 5d;
    proxy_cache_valid any 5d;
    proxy_cache_key '$host:$server_port$request_uri';
    add_header X-Cache '$upstream_cache_status from $host';
    proxy_pass http://192.168.8.203:8080;
    expires 30d;
}

Cache files are stored in /dev/shm (tmpfs) for fast in‑memory access.

SSL Configuration

Generate a self‑signed CA and server certificates using OpenSSL, then configure Nginx to use them.

# Create CA certificate
openssl genrsa -des3 -out ca.key 2048
openssl req -new -x509 -days 7305 -key ca.key -out ca.crt
# Create server certificate signed by the CA
openssl genrsa -des3 -out client.key 1024
openssl req -new -key client.key -out client.csr
openssl x509 -req -in client.csr -out client.pem -signkey client.key -CA ca.crt -CAkey ca.key -CAcreateserial -days 3650
# Remove password from key for Nginx
openssl rsa -in client.key -out client.key.unsecure

server {
    server_name localhost;
    listen 443 ssl;
    root html;
    location / {
        index index.html index.html;
    }
    ssl on;
    ssl_certificate keys/client.pem;
    ssl_certificate_key keys/client.key.unsecure;
}

To force HTTP to HTTPS, add a simple redirect server:

server {
    server_name localhost;
    listen 80;
    rewrite ^(.*)$ https://$host$1 permanent;
}

Key Points

Define low‑privilege user and limit worker resources.

Use upstream blocks for load balancing static and dynamic back‑ends.

Separate static files from Tomcat to improve performance.

Enable gzip to compress responses.

Configure proxy_cache with a shared memory path for fast caching.

Set up self‑signed SSL certificates and redirect HTTP to HTTPS.