大数据系列 | 使用Nginx作为数据采集接收端

1. 编译安装Nginx

   Nginx依赖包安装：

root@nginx:~# apt install -y vim lua-devel gcc gcc-c++ autoconf automake zlib zlib-devel openssl openssl-devel gcc zlib1g-dev libpcre3 libpcre3-dev  openssl libssl-dev

root@nginx:/opt# tar xf lua-nginx-module-0.10.13.tar.gz     // 网络上可以下载
root@nginx:/opt# tar xf ngx_devel_kit-0.3.0.tar.gz			// 网络上可以下载
root@nginx:/opt# git clone https://luajit.org/git/luajit.git
root@nginx:/opt/luajit# make PREFIX=/opt/LuaJIT
==== Building LuaJIT 2.1 ====
......(省略部分)......
OK        Successfully built LuaJIT
make[1]: Leaving directory '/opt/luajit/src'
==== Successfully built LuaJIT 2.1 ====

root@nginx:/opt/luajit# make install PREFIX=/opt/LuaJIT
......(省略部分)......
==== Successfully installed LuaJIT 2.1.1710088188 to /opt/LuaJIT ====

root@nginx:/opt# vim /etc/profile
export LUAJIT_LIB=/opt/LuaJIT/lib
export LUAJIT_INC=/opt/LuaJIT/include/luajit-2.1
root@nginx:/opt# source /etc/profile

luajit安装官方手册：https://luajit.org/install.html

   编译安装Nginx：

root@nginx:~# useradd -s /sbin/nologin -M nginx

root@nginx:~# tar xf nginx-1.16.1.tar.gz
root@nginx:~# mv /root/nginx-1.16.1 /root/nginx
root@nginx:~# cd nginx
root@nginx:~/nginx# ./configure  \
--prefix=/opt/nginx/ \
--sbin-path=/opt/nginx/sbin/nginx \
--conf-path=/opt/nginx/conf/nginx.conf \
--error-log-path=/opt/nginx/logs/error.log \
--http-log-path=/opt/nginx/logs/access.log \
--pid-path=/opt/nginx/logs/nginx.pid \
--lock-path=/opt/nginx/logs/nginx.lock \
--user=nginx \
--group=nginx \
--with-http_ssl_module \
--with-http_realip_module \
--with-http_addition_module \
--with-http_sub_module \
--with-http_dav_module \
--with-http_flv_module \
--with-http_mp4_module \
--with-http_gunzip_module \
--with-http_gzip_static_module \
--with-http_random_index_module \
--with-http_secure_link_module \
--with-http_stub_status_module \
--with-http_auth_request_module \
--with-stream --with-stream_ssl_module \
--add-module=/opt/ngx_devel_kit-0.3.0 \
--add-module=/opt/lua-nginx-module-0.10.13

root@nginx:~# make && make install

   Ubuntu22.04编译安装nginx时报错error: ‘ENGINE_free’ is deprecated: Since OpenSSL 3.0
   加上：./configure --with-cc-opt="-Wno-error -Wno-deprecated-declarations"

[root@localhost ~]# /opt/nginx/sbin/nginx -V
nginx version: nginx/1.16.1
built by gcc 4.8.5 20150623 (Red Hat 4.8.5-39) (GCC)
built with OpenSSL 1.0.2k-fips  26 Jan 2017
TLS SNI support enabled
configure arguments: --prefix=/opt/nginx/ --sbin-path=/opt/nginx/sbin/nginx --conf-path=/opt/nginx/conf/nginx.conf --error-log-path=/opt/nginx/logs/error.log --http-log-path=/opt/nginx/logs/access.log --pid-path=/opt/nginx/logs/nginx.pid --lock-path=/opt/nginx/logs/nginx.lock --user=nginx --group=nginx --with-http_ssl_module --with-http_realip_module --with-http_addition_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_random_index_module --with-http_secure_link_module --with-http_stub_status_module --with-http_auth_request_module --with-stream --with-stream_ssl_module --add-module=/opt/ngx_devel_kit-0.3.0 --add-module=/opt/lua-nginx-module-0.10.13

   编辑nginx.conf配置文件：

root@nginx:~# cat /opt/nginx/conf/nginx.conf
user  nginx;
worker_processes  42;

error_log  logs/error.log;

pid        logs/nginx.pid;

events {

    use epoll;
    multi_accept on;
    worker_connections  65535;
}

http {
    include       mime.types;
    default_type  text/html;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    log_format  demo  escape=json 'ip=$remote_addr&$request_body';

    log_format  test  escape=json '$remote_addr | $http_x_forwarded_for |  $time_local | $request | $uri | '
                      '$status | $body_bytes_sent | $bytes_sent | $gzip_ratio | $http_referer | '
                      '$http_user_agent | $http_Authorization | $upstream_addr | $upstream_response_time | $upstream_status | $request_time | $request_body | $http_host';



    keepalive_timeout  120;
    keepalive_requests 10000;
    server_names_hash_bucket_size 128;
    client_header_buffer_size     32k;
    large_client_header_buffers 4 32k;

    fastcgi_buffers 32 8k;
    client_body_buffer_size 1024k;
    server_tokens off;

    gzip  on;
    gzip_http_version 1.0;
    gzip_min_length 1100;
    gzip_buffers    4 16k;
    gzip_comp_level 9;
    gzip_vary on;
    gzip_types  text/plain application/x-javascript application/javascript text/css application/xml text/javascript;

    output_buffers  1 32k;
    postpone_output 1460;
    add_header X-Cachei $upstream_cache_status;


  include /opt/nginx/conf/vhost/*.conf;

}

   编辑vhost文件：

root@nginx:~#  mkdir -p /opt/nginx/logs/test/

root@nginx:~# mkdir /opt/nginx/conf/vhost/
root@nginx:~# vim /opt/nginx/conf/vhost/172.26.160.14.conf
upstream wangzongyu {
  server 10.1.248.197:81 weight=10 max_fails=2 fail_timeout=10s;
  keepalive 300;
}


server {
        listen       80;
        server_name  172.26.160.14;
        include    /opt/nginx/conf/proxy.conf;


        location / {
            proxy_pass http://wangzongyu/;
            content_by_lua '
                ngx.header.content_type = "text/html";
                ngx.req.read_body()
            ';

            proxy_http_version 1.1;
            proxy_set_header Connection "";

            access_log     /opt/nginx/logs/test/wangzongyu.log test;
            error_log      /opt/nginx/logs/error.wangzongyu.log;
        }

        location  = /ngx_status {


        stub_status on;

        access_log off;

        allow all;

        #deny all;
      }
}

server {
      listen  81;

      location  = / {
        root /opt/nginx/html;
        index  index.html index.htm;
        access_log off;

        #return 200;

      }

}

   启动Nginx：

root@nginx:~# /opt/nginx/sbin/nginx -t
nginx: the configuration file /opt/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /opt/nginx/conf/nginx.conf test is successful

root@nginx:~#  /opt/nginx/sbin/nginx

root@nginx:~# cat /opt/nginx/logs/test/wangzongyu.log
192.168.223.63 |  |  20/Mar/2024:08:37:53 +0000 | GET / HTTP/1.1 | / | 200 | 31 | 226 | 0.00 |  | Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 |  |  |  |  | 0.000 |  | 172.26.160.14

2. 通过程序对Nginx上传日志