1.4、FastDFS + Nginx 安装说明（高可用）

FastDFS + Nginx 安装说明（高可用）

实现统一的对外下载访问入口的高可用架构，其中所有的Nginx只做下载用途，上传通过tracker进行上传

一、环境准备

1.1 系统软件说明：

1.2 所需环境资源及用途：

1.3 指定阿里yum源

$ sudo mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
$ sudo wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
$ sudo yum makecache

1.4 准备安装包

下载 libfastcommon、fastdfs、fastdfs-nginx-module

下载的包都存放在 /www/fastdfs 目录下

[root@fastdfs ~]# mkdir /www/fastdfs
[root@fastdfs ~]# cd /www/fastdfs
[root@fastdfs ~]# ls
fastdfs-5.11.zip  fastdfs-nginx-module-1.20.zip  libfastcommon-1.0.39.zip  nginx-1.12.2.tar.gz

版本对照：

二、服务安装：

2.1 安装依赖组件：

##安装编译工具 gcc
[root@fastdfs ~]# yum install -y gcc gcc-c++
##安装PCRE库，pcre提供编译版本的库，pcre-devel提供开发阶段的头文件和编译项目的源代码。
[root@fastdfs ~]# yum install -y pcre pcre-devel
##安装OpenSSL加密库，nginx中如果服务器需要提供安全网页则会用到OpenSSL库
[root@fastdfs ~]# yum install -y openssl openssl-devel
##安装zlib库，zlib库提供了压缩算法，在nginx的各种模块中需要使用gzip压缩
[root@fastdfs ~]# yum install -y zlib zlib-devel
[root@fastdfs ~]# yum install -y libevent libevent-devel perl unzip net-tools wget

2.2 安装 libfastcommon

[root@fastdfs ~]# cd /www/fastdfs
[root@fastdfs ~]# ls
fastdfs-5.11.zip  fastdfs-nginx-module-1.20.zip  libfastcommon-1.0.39.zip  nginx-1.12.0.zip
[root@fastdfs ~]# unzip libfastcommon-1.0.39.zip
[root@fastdfs libfastcommon-1.0.39]# ./make.sh
[root@fastdfs libfastcommon-1.0.39]# ./make.sh install

#执行完成后创建软链接：
[root@fastdfs libfastcommon-1.0.39]# ln -s /usr/lib64/libfastcommon.so /usr/local/lib/libfastcommon.so 
[root@fastdfs libfastcommon-1.0.39]# ln -s /usr/lib64/libfastcommon.so /usr/lib/libfastcommon.so 
[root@fastdfs libfastcommon-1.0.39]# ln -s /usr/lib64/libfdfsclient.so /usr/local/lib/libfdfsclient.so 
[root@fastdfs libfastcommon-1.0.39]# ln -s /usr/lib64/libfdfsclient.so /usr/lib/libfdfsclient.so

2.3 安装 FastDFS

[root@fastdfs ~]# cd /www/fastdfs
[root@fastdfs ~]# ls
fastdfs-5.11.zip  fastdfs-nginx-module-1.20.zip  libfastcommon-1.0.39.zip  nginx-1.12.2.tar.gz
[root@fastdfs ~]# unzip fastdfs-5.11.zip
[root@fastdfs ~]# cd fastdfs-5.11/
[root@fastdfs fastdfs-5.11]# ./make.sh
[root@fastdfs fastdfs-5.11]# ./make.sh install

#安装好后，程序是在/usr/bin目录下：
[root@fastdfs fastdfs-5.11]# which fdfs_trackerd
/usr/bin/fdfs_trackerd

#默认配置文件在/etc/fdfs目录下：
[root@fastdfs fastdfs-5.11]# ls /etc/fdfs
client.conf.sample storage_ids.conf.sample  tracker.conf.sample storage.conf.sample

#拷贝默认配置文件：
[root@fastdfs fastdfs-5.11]# cp /etc/fdfs/tracker.conf.sample /etc/fdfs/tracker.conf #tracker节点
[root@fastdfs fastdfs-5.11]# cp /etc/fdfs/storage.conf.sample /etc/fdfs/storage.conf #storage节点
[root@fastdfs fastdfs-5.11]# cp /etc/fdfs/storage_ids.conf.sample /etc/fdfs/storage_ids.conf #storage节点ID
[root@fastdfs fastdfs-5.11]# cp /etc/fdfs/client.conf.sample /etc/fdfs/client.conf #客户端文件，测试用

#默认的配置文件不完全，而且都是模板，所以需要从fastdfs包中拷贝过来：
[root@fastdfs fastdfs-5.11]# cp /www/fastdfs/fastdfs-5.11/conf/http.conf /etc/fdfs/ #供nginx访问使用
[root@fastdfs fastdfs-5.11]# cp /www/fastdfs/fastdfs-5.11/conf/mime.types /etc/fdfs/ #供nginx访问使用

2.3.1 Tracker Server 配置：

[root@fastdfs fastdfs-5.11]# vi /etc/fdfs/tracker.conf

# 主要修改下面的几项：
# the tracker server port
port=22122
# the base path to store data and log files
base_path=/www/data/fdfs
# HTTP port on this tracker server
http.server_port=9270
# 注意：生产环境必须指定允许访问的IP,不然存在安全问题
# allow_hosts can ocur more than once, host can be hostname or ip address,
# "*" (only one asterisk) means match all ip addresses
# we can use CIDR ips like 192.168.5.64/26
# and also use range like these: 10.0.1.[0-254] and host[01-08,20-25].domain.com
# for example:
# allow_hosts=10.0.1.[1-15,20]
# allow_hosts=host[01-08,20-25].domain.com
# allow_hosts=192.168.5.64/26
allow_hosts=*

Tracker Server 随机启动：

[root@fastdfs fastdfs-5.11]# vi /lib/systemd/system/fastdfs-tracker.service

脚本内容：

[Unit]
Description=The FastDFS File server
After=network.target remote-fs.target nss-lookup.target

[Service]
Type=forking
ExecStart=/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf start
ExecStop=/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf stop
ExecRestart=/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf restart

[Install]
WantedBy=multi-user.target

设置随机启动：

[root@fastdfs fastdfs-5.11]# systemctl daemon-reload
[root@fastdfs fastdfs-5.11]# systemctl enable fastdfs-tracker.service
[root@fastdfs fastdfs-5.11]# systemctl start fastdfs-tracker.service
[root@fastdfs fastdfs-5.11]# netstat -tulnp #查看服务是否启动，端口是否打开

2.3.2 Storage Server 配置

[root@fastdfs fastdfs-5.11]# vi /etc/fdfs/storage.conf

# 主要修改下面的几项：

# storage所属的组
group_name=group1
# the storage server port
port=23000
# the base path to store data and log files
base_path=/www/data/fdfs
# store_path#, based 0, if store_path0 not exists, it's value is base_path
# the paths must be exist
store_path0=/www/data/fdfs
#store_path1=/www/data/fdfs2
# tracker服务器，虽然是同一台机器上，但是不能写127.0.0.1。这项配置可以出现一次或多次
tracker_server=192.168.2.200:22122 # tracker服务器IP和端口
tracker_server=192.168.2.201:22122 # tracker服务器IP和端口
# the port of the web server on this storage server
http.server_port=8888
# 注意：生产环境必须指定允许访问的IP,不然存在安全问题
# allow_hosts can ocur more than once, host can be hostname or ip address,
# "*" (only one asterisk) means match all ip addresses
# we can use CIDR ips like 192.168.5.64/26
# and also use range like these: 10.0.1.[0-254] and host[01-08,20-25].domain.com
# for example:
# allow_hosts=10.0.1.[1-15,20]
# allow_hosts=host[01-08,20-25].domain.com
# allow_hosts=192.168.5.64/26
allow_hosts=*

Storage Server 随机启动：

[root@fastdfs fastdfs-5.11]# vi /lib/systemd/system/fastdfs-storage.service

脚本内容：

[Unit]
Description=The FastDFS File server
After=network.target remote-fs.target nss-lookup.target

[Service]
Type=forking
ExecStart=/usr/bin/fdfs_storaged /etc/fdfs/storage.conf start
ExecStop=/usr/bin/fdfs_storaged /etc/fdfs/storage.conf stop
ExecRestart=/usr/bin/fdfs_storaged /etc/fdfs/storage.conf restart

[Install]
WantedBy=multi-user.target

设置随机启动：

[root@fastdfs fastdfs-5.11]# systemctl daemon-reload
[root@fastdfs fastdfs-5.11]# systemctl enable fastdfs-storage.service
[root@fastdfs fastdfs-5.11]# systemctl start fastdfs-storage.service
[root@fastdfs fastdfs-5.11]# netstat -tulnp #查看服务是否启动，端口是否打开

查看集群状态：

[root@fastdfs fastdfs-5.11]# fdfs_monitor /etc/fdfs/storage.conf list

2.3.4 Client 配置

[root@fastdfs fastdfs-5.11]# mkdir -p /www/data/fdfs/client
[root@fastdfs fastdfs-5.11]# vi /etc/fdfs/client.conf

# 主要修改下面的几项：

# the base path to store log files
base_path=/www/data/fdfs/client
# tracker_server can ocur more than once, and tracker_server format is
#  "host:port", host can be hostname or ip address
tracker_server=192.168.2.200:22122 # tracker服务器IP和端口
tracker_server=192.168.2.201:22122 # tracker服务器IP和端口
#HTTP settings
http.tracker_server_port=9270

保存后测试,返回ID表示成功 如：group1/M00/00/00/xx.tar.gz

[root@fastdfs fastdfs-5.11]# fdfs_upload_file /etc/fdfs/client.conf README.md
group1/M00/00/00/wKgCyF0bG2KAeOVmAAAJTMk6Vgo4510.md

配置过程中有几点要注意：

• 确保配置中用到的目录已经创建了。比如~/fdfs/client、~/fdfs/data、~/fdfs/logs

• 确保各种配置文件之间引用的端口一直。比如：

  mod_fastdfs.conf文件中tracker_server的端口应该跟tracker.conf中port一致；
  mod_fastdfs.conf文件中storage_server_port的端口应该跟跟storage.conf中port一致；

• 其他配置或文件虽然不用修改，但是fastdfs-nginx-module模块会用到：

  anti-steal.jpg
http.conf
mime.types

2.3.5 安装 nginx 和 fastdfs-nginx-module

解压并配置 fastdfs-nginx-module

[root@fastdfs ~]# cd /www/fastdfs
[root@fastdfs ~]# ls
fastdfs-5.11.zip  fastdfs-nginx-module-1.20.zip  libfastcommon-1.0.39.zip  nginx-1.12.2.tar.gz
[root@fastdfs ~]# unzip fastdfs-nginx-module-1.20.zip
[root@fastdfs ~]# cp /www/fastdfs/fastdfs-nginx-module-1.20/src/mod_fastdfs.conf /etc/fdfs
[root@fastdfs ~]# vi /etc/fdfs/mod_fastdfs.conf
# 主要修改下面的几项：

# connect timeout in seconds
# default value is 30s
connect_timeout=5

# the base path to store log files
base_path=/www/data/fdfs

# FastDFS tracker_server can ocur more than once, and tracker_server format is
#  "host:port", host can be hostname or ip address
# valid only when load_fdfs_parameters_from_tracker is true
tracker_server=192.168.2.200:22122 # tracker服务器IP和端口
tracker_server=192.168.2.200:22122 # tracker服务器IP和端口

# the port of the local storage server
# the default value is 23000
storage_server_port=23000

# if the url / uri including the group name
# set to false when uri like /M00/00/00/xxx
# set to true when uri like ${group_name}/M00/00/00/xxx, such as group1/M00/xxx
# default value is false
url_have_group_name = true

# path(disk or mount point) count, default value is 1
# must same as storage.conf
store_path_count=1

# store_path#, based 0, if store_path0 not exists, it's value is base_path
# the paths must be exist
# must same as storage.conf
store_path0=/www/data/fdfs
#store_path1=/www/data/fdfs1

安装 Nginx

[root@fastdfs ~]# cd /www/fastdfs
[root@fastdfs ~]# tar -zxvf nginx-1.12.2.tar.gz
[root@fastdfs ~]# cd nginx-1.12.2/
[root@fastdfs nginx-1.12.2]# ./configure \
--prefix=/usr/local/nginx \
--conf-path=/etc/nginx/nginx.conf \
--http-log-path=/var/log/nginx/access.log \
--error-log-path=/var/log/nginx/error.log \
--lock-path=/var/lock/nginx.lock \
--pid-path=/run/nginx.pid \
--http-client-body-temp-path=/var/lib/nginx/body \
--http-fastcgi-temp-path=/var/lib/nginx/fastcgi \
--http-proxy-temp-path=/var/lib/nginx/proxy \
--http-scgi-temp-path=/var/lib/nginx/scgi \
--http-uwsgi-temp-path=/var/lib/nginx/uwsgi \
--with-debug \
--with-pcre-jit \
--with-http_ssl_module \
--with-http_stub_status_module \
--with-http_realip_module \
--with-http_auth_request_module \
--with-http_addition_module \
--with-http_dav_module \
--with-http_geoip_module \
--with-http_gunzip_module \
--with-http_gzip_static_module \
--with-http_image_filter_module \
--with-http_v2_module \
--with-http_sub_module \
--with-http_xslt_module \
--with-stream \
--with-stream_ssl_module \
--with-mail \
--with-mail_ssl_module \
--with-threads \
--add-module=/www/fastdfs/fastdfs-nginx-module-1.20/src

执行完成后会输出信息，通过这些信息可以知道编译出来的nginx会装到哪里，配置文件会放在哪里，错误日志会放在哪里：

Configuration summary
  + using threads
  + using system PCRE library
  + using system OpenSSL library
  + using system zlib library

  nginx path prefix: "/usr/local/nginx"
  nginx binary file: "/usr/local/nginx/sbin/nginx"
  nginx modules path: "/usr/local/nginx/modules"
  nginx configuration prefix: "/etc/nginx"
  nginx configuration file: "/etc/nginx/nginx.conf"
  nginx pid file: "/run/nginx.pid"
  nginx error log file: "/var/log/nginx/error.log"
  nginx http access log file: "/var/log/nginx/access.log"
  nginx http client request body temporary files: "/var/lib/nginx/body"
  nginx http proxy temporary files: "/var/lib/nginx/proxy"
  nginx http fastcgi temporary files: "/var/lib/nginx/fastcgi"
  nginx http uwsgi temporary files: "/var/lib/nginx/uwsgi"
  nginx http scgi temporary files: "/var/lib/nginx/scgi"

继续执行 make 和 make install ：

[root@fastdfs nginx-1.12.2]# make && make install

配置 nginx.config

[root@fastdfs nginx-1.12.2]# vi /etc/nginx/nginx.conf

#增加如下内容
    server {
        listen 8888; ## 该端口为storage.conf中的http.server_port相同
        server_name localhost;
        location ~/group[0-9]/ {
            if ($arg_attname ~ "^(.*).apk") {
                add_header Content-Disposition "attachment;filename=$arg_attname";
            }        
            root /www/data/fdfs;
            ngx_fastdfs_module;
        }
        error_page 500 502 503 504 /50x.html;
        location = /50x.html {
            root html;
        }
    }

说明：

（1）”user root”是解决下载操作时报404的问题

（2）8888端口号与/etc/fdfs/storage.conf中的http.server_port=8888相对应

（3）storage对应有多个group的情况下，访问路径带group名称，例如：/group1/M00/00/00/**，对应nginx配置：

location ~/group[0-9]/ {
    if ($arg_attname ~ "^(.*).apk") {
        add_header Content-Disposition "attachment;filename=$arg_attname";
    }
    root /www/data/fdfs;
    ngx_fastdfs_module;
}

配置nginx开机启动

[root@fastdfs nginx-1.12.2]# vi /lib/systemd/system/nginx.service

文件内容如下：

[Unit]
Description=nginx
After=network.target

[Service]
Type=forking
ExecStart=/usr/local/nginx/sbin/nginx
ExecReload=/usr/local/nginx/sbin/nginx reload
ExecStop=/usr/local/nginx/sbin/nginx stop
PrivateTmp=true

[Install]
WantedBy=multi-user.target

退出并保存文件，执行systemctl enable nginx.service使nginx开机启动

[root@fastdfs fastdfs-5.11]# systemctl daemon-reload
[root@fastdfs nginx-1.12.2]# systemctl enable nginx.service
[root@fastdfs nginx-1.12.2]# systemctl start nginx.service    启动nginx
[root@fastdfs nginx-1.12.2]# systemctl stop nginx.service    结束nginx
[root@fastdfs nginx-1.12.2]# systemctl restart nginx.service    重启nginx

验证是否安装成功

[root@fastdfs nginx-1.12.2]# ps -ef|grep nginx
root      7758     1  0 21:20 ?        00:00:00 nginx: master process /usr/local/nginx/sbin/nginx
root      7759  7758  0 21:20 ?        00:00:00 nginx: worker process
root      7863   845  0 21:20 pts/1    00:00:00 grep --color=auto nginx

输入http://服务器IP/ 如果能看到nginx的界面，就表示安装成功了

如果不能访问，关闭防火墙：

[root@fastdfs nginx-1.12.2]# systemctl start firewalld    启动
[root@fastdfs nginx-1.12.2]# systemctl stop firewalld    关闭
[root@fastdfs nginx-1.12.2]# systemctl status firewalld    查看状态
[root@fastdfs nginx-1.12.2]# systemctl disable firewalld    开机禁用
[root@fastdfs nginx-1.12.2]# systemctl enable firewalld    开机启用

上传图片到fastdfs：

[root@fastdfs ~]# fdfs_upload_file /etc/fdfs/client.conf test.png
group1/M00/00/00/wKgCyF0bWrOAMmJDAAD4TDHQw7Y274.png

浏览器访问：http://192.168.2.200:8888/group1/M00/00/00/wKgCyF0bWrOAMmJDAAD4TDHQw7Y274.png

2.3.6 在nginx的构建中会遇到不少的报错，具体如下：

• ./configure: error: the Google perftools module requires the Google perftools library. You can either do not enable the module or install the library.
  解决方法如下：

[root@fastdfs nginx-1.12.2]# yum install -y gperftools

• /configure: error: the HTTP rewrite module requires the PCRE library.
  解决方法如下：

[root@fastdfs nginx-1.12.2]# yum -y install pcre pcre-devel

• ./configure: error: the HTTP cache module requires md5 functions from OpenSSL library. You can either disable the module by using –without-http-cache option, or install the OpenSSL library into the system, or build the OpenSSL library statically from the source with nginx by using –with-http_ssl_module –with-openssl= options.
  解决方法如下：

[root@fastdfs nginx-1.12.2]# yum -y install openssl openssl-devel

• ./configure: error: the HTTP gzip module requires the zlib library. You can either disable the module by using –without-http_gzip_module option, or install the zlib library into the system, or build the zlib library statically from the source with nginx by using –with-zlib= option.
  解决方法如下：

[root@fastdfs nginx-1.12.2]#  yum install -y zlib-devel

• ./configure: error: the HTTP XSLT module requires the libxml2/libxslt libraries. You can either do not enable the module or install the libraries.
  解决方法如下：

[root@fastdfs nginx-1.12.2]# yum -y install libxml2 libxml2-dev libxslt-dev
[root@fastdfs nginx-1.12.2]# yum -y install libxslt-devel

• ./configure: error: the HTTP image filter module requires the GD library. You can either do not enable the module or install the libraries.
  解决方法如下：

[root@fastdfs nginx-1.12.2]# yum -y install gd-devel

• ./configure: error: perl module ExtUtils::Embed is required
  解决方法如下：

[root@fastdfs nginx-1.12.2]# yum -y install perl-devel perl-ExtUtils-Embed

• ./configure: error: the GeoIP module requires the GeoIP library. You can either do not enable the module or install the library.
  解决方法如下：

[root@fastdfs nginx-1.12.2]# yum -y install GeoIP GeoIP-devel GeoIP-data

• 在make过程中会出现一个报错：/usr/include/fastdfs/fdfs_define.h:15:27: fatal error: common_define.h: No such file or directory
  解决方法如下：

修改 fastdfs-nginx-module-1.20/src/config 文件，修改如下：
ngx_module_incs="/usr/include/fastdfs /usr/include/fastcommon/"
CORE_INCS="$CORE_INCS /usr/include/fastdfs /usr/include/fastcommon/"
然后重新 ./configure && make，就可以了

2.3.7 配置文件访问的负载均衡和高可用

在192.168.0.100和101上安装nginx、keepalived

yum install -y nginx keepalived

nginx的配置文件如下：

upstream fdfs_group01 {
    server 192.168.0.10:8888 weight=1 max_fails=2 fail_timeout=30s;
    server 192.168.0.11:8888 weight=1 max_fails=2 fail_timeout=30s;
}
server {
    listen 80;
    server_name localhost;
    location /group01{
        proxy_next_upstream http_502 http_504 error timeout invalid_header;
        proxy_pass http://fdfs_group01;
        expires 30d;
    }
}

keepalived配置文件如下：

global_defs {
   router_id LVS_DEVEL
}

vrrp_script chk_ngx {
    script "/etc/keepalived/check_nginx.sh"
    interval 2
    weight -5
    fall 3  
    rise 2
}


vrrp_instance VI_1 {
    interface eno16777984
    state MASTER
    priority 100
    virtual_router_id 11
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }

    unicast_src_ip 192.168.0.100

    unicast_peer {
        192.168.0.101
    }

    virtual_ipaddress {
        192.168.0.200
    }

    track_script {
       chk_ngx
    }
    notify_master "/etc/keepalived/notify.sh master"  
    notify_backup "/etc/keepalived/notify.sh backup"  
    notify_fault "/etc/keepalived/notify.sh fault"
}

check_nginx.sh

#!/bin/bash

counter=$(ps -C nginx --no-heading|wc -l)
if [ "${counter}" = "0" ]; then
    exit 1
else
    exit 0
fi

2.3.7 FastDFS 文件防盗

FastDFS 配置 Nginx 负载后，默认情况只要知道访问地址就能获取文件，这种情况是极其不安全的，需要开启FastDFS自带的token安全机制，来实现文件防盗功能。

[root@fastdfs ~]# vi /etc/fdfs/http.conf

下面配置主要需要配置 check_token=true 来开启token验证, 指定Token超时时间token_ttl=900, 设置安全密钥secret_key=6fGkkBwRnaDjfXdh，指定异常情况返回结果token_check_fail=提示图片

# HTTP default content type
http.default_content_type = application/octet-stream

# MIME types mapping filename
# MIME types file format: MIME_type  extensions
# such as:  image/jpeg    jpeg jpg jpe
# you can use apache's MIME file: mime.types
http.mime_types_filename=mime.types

# if use token to anti-steal
# default value is false (0)
http.anti_steal.check_token=true

# token TTL (time to live), seconds
# default value is 600
http.anti_steal.token_ttl=900

# secret key to generate anti-steal token
# this parameter must be set when http.anti_steal.check_token set to true
# the length of the secret key should not exceed 128 bytes
http.anti_steal.secret_key=6fGkkBwRnaDjfXdh

# return the content of the file when check token fail
# default value is empty (no file sepecified)
http.anti_steal.token_check_fail=/www/data/fdfs/anti-steal.jpg

# if support multi regions for HTTP Range
# default value is true
http.multi_range.enabed = true

参考文档：

https://segmentfault.com/a/1190000018251300?utm_source=tag-newest
https://www.cnblogs.com/sunnydou/p/49b92d511047f4f9da6cd727cfd415d5.html
https://github.com/happyfish100/fastdfs/wiki
https://www.cnblogs.com/youzhibing/p/9187765.html