音视频

基于Gstreamer的rtp转rtmp代码

2019-11-23 – 01:20

rtp2rtmp_video.c
========

1. 推流到rtmp服务器

1
2
3

 
$ ffmpeg -re -i ./fulankelin-hd.mp4 -an -vcodec h264 -f rtp rtp://127.0.0.1:5004 -vn -acodec libopus -f rtp rtp://127.0.0.1:5003
$

2. SDP信息

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

SDP:
 
SDP:
v=0
o=- 0 0 IN IP4 127.0.0.1
s=No Name
t=0 0
a=tool:libavformat 58.29.100
m=video 5004 RTP/AVP 96
c=IN IP4 127.0.0.1
a=rtpmap:96 H264/90000
a=fmtp:96 packetization-mode=1
m=audio 5003 RTP/AVP 97
c=IN IP4 127.0.0.1
b=AS:96
a=rtpmap:97 opus/48000/2
a=fmtp:97 sprop-stereo=1
 
.

3. gst-launch测试

1
2
3
4
5
6
7
8
9
10

 
gst-launch-1.0 -em \
  rtpbin name=rtpbin latency=5 \
  udpsrc port=5003 caps="application/x-rtp,media=(string)audio,clock-rate=(int)48000,encoding-name=(string)OPUS" ! rtpbin.recv_rtp_sink_0 \
    rtpbin.  ! rtpopusdepay ! opusdec ! audioconvert ! audioresample ! avenc_aac ! mux. \
  udpsrc port=5004 caps="application/x-rtp,media=(string)video,clock-rate=(int)90000,encoding-name=(string)H264" ! rtpbin.recv_rtp_sink_1 \
    rtpbin.  ! rtph264depay ! h264parse ! mux. \
  flvmux name=mux streamable=true ! rtmpsink sync=false location=rtmp://u1802/live/demo
 
.

4. 程序代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183

 
#include <string.h>
#include <math.h>
 
#include <gst/gst.h>
 
#define VIDEO_CAPS "application/x-rtp,media=(string)video,clock-rate=(int)90000,encoding-name=(string)H264"
#define AUDIO_CAPS "application/x-rtp,media=(string)audio,clock-rate=(int)48000,encoding-name=(string)OPUS"
 
/* will be called when rtpbin has validated a payload that we can depayload */
static void
pad_added_cb(GstElement *rtpbin, GstPad *new_pad, GstElement *depay)
{
    char *pad_name = GST_PAD_NAME(new_pad);
    char *depay_name = gst_element_get_name(depay);
    if (strstr(pad_name, "recv_rtp_src_0_") && strstr(depay_name, "audiodepay"))
    {
        GstPad *sinkpad;
        GstPadLinkReturn lres;
 
        g_print("new payload on rtpbin: %s %s %s\n",
                gst_element_get_name(rtpbin), GST_PAD_NAME(new_pad), gst_element_get_name(depay));
 
        sinkpad = gst_element_get_static_pad(depay, "sink");
        g_assert(sinkpad);
 
        lres = gst_pad_link(new_pad, sinkpad);
        g_assert(lres == GST_PAD_LINK_OK);
        gst_object_unref(sinkpad);
    }
    else if (strstr(pad_name, "recv_rtp_src_1_") && strstr(depay_name, "videodepay"))
    {
        GstPad *sinkpad;
        GstPadLinkReturn lres;
 
        g_print("new payload on rtpbin: %s %s %s\n",
                gst_element_get_name(rtpbin), GST_PAD_NAME(new_pad), gst_element_get_name(depay));
 
        sinkpad = gst_element_get_static_pad(depay, "sink");
        g_assert(sinkpad);
 
        lres = gst_pad_link(new_pad, sinkpad);
        g_assert(lres == GST_PAD_LINK_OK);
        gst_object_unref(sinkpad);
    }
}
 
int main(int argc, char *argv[])
{
    GMainLoop *loop;
    GstElement *pipeline;
 
    GstElement *rtpbin;
    GstElement *audiosrc, *audiodepay, *audiodec, *audiores, *audioconv, *audiosink;
    GstElement *videosrc, *videodepay, *videosink;
    GstElement *flvmux, *rtmpsink;
 
    gboolean res;
    GstCaps *caps;
    GstPadLinkReturn lres;
    GstPad *srcpad, *audio_sinkpad, *video_sinkpad;
 
    gst_init(&argc, &argv);
    pipeline = gst_pipeline_new(NULL);
    g_assert(pipeline);
 
    /* the rtpbin element */
    rtpbin = gst_element_factory_make("rtpbin", "rtpbin");
    g_assert(rtpbin);
    gst_bin_add(GST_BIN(pipeline), rtpbin);
    // 001 源
    audiosrc = gst_element_factory_make("udpsrc", "audiosrc");
    g_assert(audiosrc);
    g_object_set(audiosrc, "port", 5003, NULL);
    caps = gst_caps_from_string(AUDIO_CAPS);
    g_object_set(audiosrc, "caps", caps, NULL);
    gst_caps_unref(caps);
    gst_bin_add(GST_BIN(pipeline), audiosrc);
 
    videosrc = gst_element_factory_make("udpsrc", "videosrc");
    g_assert(videosrc);
    g_object_set(videosrc, "port", 5004, NULL);
    caps = gst_caps_from_string(VIDEO_CAPS);
    g_object_set(videosrc, "caps", caps, NULL);
    gst_caps_unref(caps);
    gst_bin_add(GST_BIN(pipeline), videosrc);
 
    /* now link all to the rtpbin, start by getting an RTP sinkpad for session 0 */
    srcpad = gst_element_get_static_pad(audiosrc, "src");
    audio_sinkpad = gst_element_get_request_pad(rtpbin, "recv_rtp_sink_0");
    lres = gst_pad_link(srcpad, audio_sinkpad);
    g_assert(lres == GST_PAD_LINK_OK);
    gst_object_unref(srcpad);
 
    srcpad = gst_element_get_static_pad(videosrc, "src");
    video_sinkpad = gst_element_get_request_pad(rtpbin, "recv_rtp_sink_1");
    lres = gst_pad_link(srcpad, video_sinkpad);
    g_assert(lres == GST_PAD_LINK_OK);
    gst_object_unref(srcpad);
 
    /* the depayloading and decoding */
    audiodepay = gst_element_factory_make("rtpopusdepay", "audiodepay");
    g_assert(audiodepay);
    audiodec = gst_element_factory_make("opusdec", "audiodec");
    g_assert(audiodepay);
    /* the audio playback and format conversion */
    audioconv = gst_element_factory_make("audioconvert", "audioconv");
    g_assert(audioconv);
    audiores = gst_element_factory_make("audioresample", "audiores");
    g_assert(audiores);
    audiosink = gst_element_factory_make("avenc_aac", "audiosink"); // autoaudiosink voaacenc avenc_aac avenc_opus
    g_assert(audiosink);
    /* add depayloading and playback to the pipeline and link */
    gst_bin_add_many(GST_BIN(pipeline), audiodepay, audiodec, audioconv,
                     audiores, audiosink, NULL);
    res = gst_element_link_many(audiodepay, audiodec, audioconv, audiores,
                                audiosink, NULL);
    g_assert(res == TRUE);
 
    videodepay = gst_element_factory_make("rtph264depay", "videodepay");
    g_assert(videodepay);
    videosink = gst_element_factory_make("h264parse", "videosink");
    g_assert(videosink);
    gst_bin_add_many(GST_BIN(pipeline), videodepay, videosink, NULL);
    res = gst_element_link_many(videodepay, videosink, NULL);
    g_assert(res == TRUE);
 
    // flvmux
    flvmux = gst_element_factory_make("flvmux", "flvmux");
    g_assert(flvmux);
    g_object_set(flvmux, "streamable", TRUE, NULL);
    gst_bin_add(GST_BIN(pipeline), flvmux);
 
    res = gst_element_link(audiosink, flvmux);
    g_assert(res == TRUE);
    res = gst_element_link(videosink, flvmux);
    g_assert(res == TRUE);
 
    rtmpsink = gst_element_factory_make("rtmpsink", "rtmpsink");
    g_assert(rtmpsink);
    g_object_set(rtmpsink, "sync", FALSE, NULL);
    g_object_set(rtmpsink, "location", "rtmp://u1802/live/demo2", NULL);
    gst_bin_add(GST_BIN(pipeline), rtmpsink);
    res = gst_element_link(flvmux, rtmpsink);
    g_assert(res == TRUE);
 
    /* the RTP pad that we have to connect to the depayloader will be created
   * dynamically so we connect to the pad-added signal, pass the depayloader as
   * user_data so that we can link to it. */
    g_signal_connect(rtpbin, "pad-added", G_CALLBACK(pad_added_cb), audiodepay);
    g_signal_connect(rtpbin, "pad-added", G_CALLBACK(pad_added_cb), videodepay);
 
    /* set the pipeline to playing */
    g_print("starting receiver pipeline\n");
    gst_element_set_state(pipeline, GST_STATE_PLAYING);
 
    /* we need to run a GLib main loop to get the messages */
    loop = g_main_loop_new(NULL, FALSE);
    g_main_loop_run(loop);
 
    g_print("stopping receiver pipeline\n");
    gst_element_set_state(pipeline, GST_STATE_NULL);
 
    gst_object_unref(loop);
    gst_object_unref(pipeline);
    gst_object_unref(audio_sinkpad);
    gst_object_unref(video_sinkpad);
    gst_object_unref(rtmpsink);
    gst_object_unref(flvmux);
    gst_object_unref(rtpbin);
    gst_object_unref(audiosrc);
    gst_object_unref(audiodepay);
    gst_object_unref(audiodec);
    gst_object_unref(audiores);
    gst_object_unref(audioconv);
    gst_object_unref(audiosink);
    gst_object_unref(videosrc);
    gst_object_unref(videodepay);
    gst_object_unref(videosink);
    return 0;
}
 
//
Permalink | Comment (0)
音视频

海康威视摄像头

2019-10-31 – 21:38

RTSP流格式

rtsp://[username]:[password]@[ip]:[port]/[codec]/[channel]/[subtype]/av_stream

说明：

参数 说明 示例
username 用户名 如admin
password 密码 123456
ip 设备IP 192.168.1.1
port 端口号，默认为554，不填写默认 554
codec 编码 h264,MPEG-4,mpeg4等
channel 通道号，起始为1，通道1则为ch1 ch1
subtype 码流类型，主码流为main，辅码流为sub main

推流

1
2
3

# ffmpeg -i "rtsp://username:passport@192.168.1.1:554/h264/ch1/sub/av_stream" -vcodec copy -preset:v ultrafast -tune:v zerolatency -acodec copy -f flv  -an "rtmp://192.168.1.1/live/haikang_01"
ffmpeg version 4.2.1 Copyright (c) 2000-2019 the FFmpeg developers
  built with Apple clang version 11.0.0 (clang-1100.0.33.8)
Permalink | Comment (0)
音视频

流媒体服务器nginx-rtmp安装

2019-10-31 – 17:46

下载源代码

1
2
3

# git clone git@github.com:arut/nginx-rtmp-module.git
git clone git@github.com:winshining/nginx-http-flv-module.git
axel -n 100 http://nginx.org/download/nginx-1.17.5.tar.gz

安装依赖

1
2
3
4
5
6
7
8
9

## ubintu
sudo apt-get install openssl libssl-dev
sudo apt-get install libpcre3 libpcre3-dev
sudo apt-get install zlib1g-dev
 
## centos
sudo yum install -y pcre pcre-devel
sudo yum install -y openssl openssl-devel
sudo yum install -y zlib-devel zlib

编译安装

1
2
3

tar zxvf nginx-1.17.5.tar.gz
cd nginx-1.17.5/
 ./configure --prefix=/usr/local/nginx --add-module=/home/work/nginx-http-flv-module --with-http_ssl_module --with-debug

配置nginx用户

1
2
3

sudo useradd nginx
## sudo vim /etc/passwd
## nginx:x:1001:1001:,,,:/home/nginx:/usr/sbin/nologin

创建相关用户

1
2
3
4
5
6
7
8

# 创建相关目录并修改所有者
sudo mkdir -p /usr/local/nginx/data/dash/live
sudo mkdir -p /usr/local/nginx/data/hls/live
sudo mkdir -p /usr/local/nginx/stat
sudo cp /home/work/nginx-http-flv-module/stat.xsl /usr/local/nginx/stat/
 
sudo chown -R nginx /usr/local/nginx/data
sudo chown -R nginx /usr/local/nginx/stat

修改配置文件

见附录：示例

启动nginx服务器

1
2
3

 .
 
sudo /usr/local/nginx/sbin/nginx

测试

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

.
 
# 推流
ffmpeg -re -i /Applications/ambari-vagrant/ubuntu18.4/data/fulankelin-hd.mp4 -c copy -f flv rtmp://u1802/live/fulankelin-hd
 
# 支持播放地址
rtmp://u1802/live/fulankelin-hd
 
http://u1802/live?app=live&stream=fulankelin-hd
http://u1802/live?port=1935&app=live&stream=fulankelin-hd
http://u1802/live/fulankelin-hd.flv
http://u1802/live/fulankelin-hd.flv?port=1935
 
http://u1802/live/fulankelin-hd.mpd
http://u1802/live/fulankelin-hd.m3u8
 
.

配置示例

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203

user  nginx;
worker_processes  1;
 
 
# error_log  logs/error.log;
# error_log  logs/error.log  notice;
error_log  logs/error.log  debug;
 
pid        logs/nginx.pid;
 
 
events {
    worker_connections  4096;
}
 
http {
    include       mime.types;
    default_type  application/octet-stream;
 
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
 
    access_log  logs/access.log  main;
 
    sendfile        on;
    #tcp_nopush     on;
 
    #keepalive_timeout  0;
    keepalive_timeout  65;
 
    #gzip  on;
 
    server {
        listen       80;
        server_name  localhost;
 
        #charset koi8-r;
 
        #access_log  logs/host.access.log  main;
 
        location / {
            root   html;
            index  index.html index.htm;
        }
 
        #error_page  404              /404.html;
 
        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
 
        # location ~* \.(m3u8)$ {
        #     types {
        #         application/vnd.apple.mpegurl m3u8;
        #         video/mp2t ts;
        #     }
 
        #     root /usr/local/nginx/data;
        #     add_header 'Cache-Control' 'no-cache';
        # }
        location /live {
            flv_live on; # 打开http播放flv直播流的方式
            chunked_transfer_encoding on; # 支持Transfer-Encoding: chunked方式回复
 
            add_header 'Access-Control-Allow-Origin' '*';
            add_header 'Access-Control-Allow-Credentials' 'true';
        }
 
        location ~ \.(mpd|m4a|m4v)$ {
            root /usr/local/nginx/data/dash/;
            add_header 'Cache-Control' 'no-cache';
        }
        # }
        location ~ \.(m3u8|ts)$ {
            types {
                application/vnd.apple.mpegurl m3u8;
                video/mp2t ts;
            }
 
            root /usr/local/nginx/data/hls/;
            add_header 'Cache-Control' 'no-cache';
        }
 
        location ~ \.(flv)$ {
            rewrite ^/(.*)/(.*)\.(flv)$ /$1?app=$1&stream=$2 last;
        }
 
        location /stat {
            rtmp_stat all;
            rtmp_stat_stylesheet stat.xsl;
        }
        location /stat.xsl {
            root /usr/local/nginx/stat/;
        }
 
        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}
 
        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}
 
        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }
 
 
    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;
 
    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}
 
 
    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;
 
    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;
 
    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;
 
    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;
 
    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}
 
}
 
rtmp_auto_push on;
rtmp_auto_push_reconnect 1s;
rtmp_socket_dir /tmp;
 
rtmp {
    out_queue           4096;
    out_cork            8;
    max_streams         128;
    timeout             15s;
    drop_idle_publisher 15s;
 
    log_interval 5s; #log模块在access.log中记录日志的间隔时间，对调试非常有用
    log_size     1m; #log模块用来记录日志的缓冲区大小
 
    server {
        listen 1935;
        on_connect http://127.0.0.1:3000/on_connect;
 
        application live {
            live on;
            hls on;
            hls_path /usr/local/nginx/data/hls/live;
            dash on;
            dash_path /usr/local/nginx/data/dash/live;
            gop_cache on; #打开GOP缓存，减少首屏等待时间
 
            notify_update_timeout 30s;
            notify_relay_redirect off; # 启用本地流重定向on_play和on_publish远程重定向。新的流名称是用于远程重定向的RTMP URL的MD5哈希。默认为关闭。
            notify_update_strict off; # 切换on_update回调的严格模式。默认为关闭。打开所有连接错误后，超时以及HTTP解析错误和空响应均被视为更新失败并导致连接终止。
            notify_method get;
 
            on_play http://127.0.0.1:3000/on_play;
            on_publish http://127.0.0.1:3000/on_publish;
            on_done http://127.0.0.1:3000/on_done;
            on_play_done http://127.0.0.1:3000/on_play_done;
            on_publish_done http://127.0.0.1:3000/on_publish_done;
            on_record_done http://127.0.0.1:3000/on_record_done;
            on_update http://127.0.0.1:3000/on_update;
 
        }
    }
}

问题处理

1
2
3
4
5
6
7
8
9
10
11

启动报错
nginx: [warn] 4096 worker_connections exceed open file resource limit: 1024
 
# ulimit -n 65535
 
vim /etc/security/limits.conf
* soft nofile 65535
* hard nofile 65535
 
vim /etc/sysctl.conf 
fs.file-max = 6553560
Permalink | Comment (0)
未分类

qcow2镜像定制指南

2018-03-27 – 21:20

背景

目前网络上关于定制镜像的说明很分散，需要搜寻很多文章才能完成镜像的定制任务。所以我尝试提供一个全面而系统的指南，遵循本指南，用户可以方便的完成镜像的定制。

实施步骤

一、环境配置

1、准备软件
mac pro、VmWare fusion、CentOS-7-x86_64-DVD-1708.iso、CentOS-7-x86_64-GenericCloud-1708-20180123.qcow2
2、安装嵌套CentOs环境
由于MacOs不支持Kvm，故需要在嵌套的操作系统中安装云镜像需要的软件，使用Fusion很容易在MacOs中虚拟出一个CentOs的环境。
3、修改嵌套操作系统配置
在centos关闭的情况下打开虚拟机“处理器和内存”配置，选择“高级配置”，选中“在此虚拟机中启用虚拟化管理程序”和“在此虚拟机中启用代码分析应用程序”，如无这步操作，则在启动virt-manager时会报：“
virt-manager 报 WARNING : KVM不可用.这可能是因为没有安装KVM软件包,或者没有载入KVM内核模块.您的虚拟机可能性很差。”的错误，启动虚拟机。以下操作如无特殊说明都是在嵌套操作系统中执行。
4、安装依赖

1
2
3

yum install qemu-kvm qemu-img qemu-kvm-tools qemu-kvm-common
yum install libvirt-admin libvirt-client libvirt-daemon libvirt-devel libvirt
yum install libguestfs libguestfs-tools libguestfs-bash-completion

5、编译nbd内核模块（如不使用“nbd挂载方式修改镜像”则不需要安装此模块）
执行命令，出现以下报错时，说明没有nbd模块，需要自己手动安装

1
2

modprobe nbd
modprobe: FATAL: Module nbd not found.

执行下面的命令安装hbd模块

1
2
3
4
5
6

[@xx] # cat /etc/redhat-release
CentOS Linux release 7.4.1708 (Core)
[@xx] # uname -a
Linux localhost 3.10.0-693.el7.x86_64 #1 SMP Tue Aug 22 21:09:27 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux
[] # uname -r
3.10.0-693.el7.x86_64

安装kernel组件

1

[@xx] sudo yum install kernel-devel kernel-headers

编译安装hbd组件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

# 下载对应的内核源码包
[@xx] # wget http://vault.centos.org/7.4.1708/os/Source/SPackages/kernel-3.10.0-693.el7.src.rpm
[@xx] # rpm -ihv kernel-3.10.0-693.el7.src.rpm
[@xx] # cd /root/rpmbuild/SOURCES/
[@xx] # tar Jxvf linux-3.10.0-123.el7.tar.xz -C /usr/src/kernels/
[@xx] # cd /usr/src/kernels/
# 配置源码
[@xx] # mv $(uname -r) $(uname -r)-old
[@xx] # mv linux-3.10.0-693.el7 $(uname -r)
# 编译安装
[@xx 3.10.0-693.el7.x86_64] # cd $(uname -r)
[@xx 3.10.0-693.el7.x86_64] # make mrproper
[@xx 3.10.0-693.el7.x86_64] # cp ../$(uname -r)-old/Module.symvers ./
[@xx 3.10.0-693.el7.x86_64] # cp /boot/config-$(uname -r) ./.config
[@xx 3.10.0-693.el7.x86_64] # make oldconfig
[@xx 3.10.0-693.el7.x86_64] # make prepare
[@xx 3.10.0-693.el7.x86_64] # make scripts
[@xx 3.10.0-693.el7.x86_64] # make CONFIG_BLK_DEV_NBD=m M=drivers/block
[@xx 3.10.0-693.el7.x86_64] # cp drivers/block/nbd.ko /lib/modules/$(uname -r)/kernel/drivers/block/
[@xx 3.10.0-693.el7.x86_64] # depmod -a
# 查看nbd模块
[@xx 3.10.0-693.el7.x86_64]$ modinfo nbd
filename:       /lib/modules/3.10.0-693.el7.x86_64/kernel/drivers/block/nbd.ko
license:        GPL
description:    Network Block Device
rhelversion:    7.4
srcversion:     EDE909A294AC5FE08E81957
depends:        
vermagic:       3.10.0 SMP mod_unload modversions 
parm:           nbds_max:number of network block devices to initialize (default: 16) (int)
parm:           max_part:number of partitions per device (default: 0) (int)
parm:           debugflags:flags for controlling debug output (int)

编译安装时的错误处理
阶段：make CONFIG_BLK_DEV_NBD=m M=drivers/block

1
2
3
4
5
6
7

drivers/block/nbd.c: 在函数‘__nbd_ioctl’中:
drivers/block/nbd.c:619:19: 错误：‘REQ_TYPE_SPECIAL’未声明(在此函数内第一次使用)
   sreq.cmd_type = REQ_TYPE_SPECIAL;
                   ^
drivers/block/nbd.c:619:19: 附注：每个未声明的标识符在其出现的函数内只报告一次
make[1]: *** [drivers/block/nbd.o] 错误 1
make: *** [_module_drivers/block] 错误 2

处理：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

[@xx 3.10.0-693.el7.x86_64] # vim include/linux/blkdev.h
# 由代码可知 REQ_TYPE_SPECIAL = 7
/*
 * request command types
 */
enum rq_cmd_type_bits {
        REQ_TYPE_FS             = 1,    /* fs request */
        REQ_TYPE_BLOCK_PC,              /* scsi command */
        REQ_TYPE_SENSE,                 /* sense request */
        REQ_TYPE_PM_SUSPEND,            /* suspend request */
        REQ_TYPE_PM_RESUME,             /* resume request */
        REQ_TYPE_PM_SHUTDOWN,           /* shutdown request */
#ifdef __GENKSYMS__
        REQ_TYPE_SPECIAL,               /* driver defined type */
#else
        REQ_TYPE_DRV_PRIV,              /* driver defined type */
#endif
        /*
         * for ATA/ATAPI devices. this really doesn"&gt;cmd[0] with the range of driver
         * private REQ_LB opcodes to differentiate what type of request this is
         */
        REQ_TYPE_ATA_TASKFILE,
        REQ_TYPE_ATA_PC,
};
# 修改nbd.c文件
[@xx 3.10.0-693.el7.x86_64] # vim drivers/block/nbd.c
# sreq.cmd_type = REQ_TYPE_SPECIAL;
sreq.cmd_type = 7;
# 重新执行命令
[@xx 3.10.0-693.el7.x86_64] # make CONFIG_BLK_DEV_NBD=m M=drivers/block

二、设置镜像共享

设置嵌套虚拟机文件夹共享
qcow2文件放置在mac本地文件夹中，嵌套虚拟机通过文件共享的方式使用qcow2文件。需要注意的是qcow2文件权限需要在macos中设置为可读写，否则在嵌套虚拟机中无法更新配置。

1

[mac@] # chmod 755 ./CentOS-7-x86_64-GenericCloud-1708-20180123.qcow2

嵌套虚拟机中，需要要关闭SeLinux否则同样无法更新镜像内容

1

sudo /usr/sbin/setenforce 0

三、guestfish工具使用

1、示例程序：获取镜像ip地址

1
2
3
4
5
6

guestfish --rw -a ./CentOS-7-x86_64-GenericCloud-1708-20180123.qcow2
# run
# list-filesystems
/dev/sda1: xfs
# mount /dev/sda1 /
# vi /var/log/cloud-init.log

2、示例程序：配置用户访问权限

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

guestfish --rw -a ./CentOS-7-x86_64-GenericCloud-1708-20180123.qcow2
# run
# list-filesystems
/dev/sda1: xfs
# mount /dev/sda1 /
# mkdir /home/dc2-user/.ssh
# 注意guestfish与shell的区别，权限位是四位
# chown 1001 1001 /home/dc2-user/.ssh
# chmod 0700 /home/dc2-user/.ssh
# touch /home/dc2-user/.ssh/authorized_keys
# chmod 0600 /home/dc2-user/.ssh/authorized_keys
# chown 1001 1001 /home/dc2-user/.ssh/authorized_keys
# ll /home/dc2-user/.ssh/authorized_keys
-rw------- 1 1001 1001 0 Mar  1 10:05 /sysroot/home/dc2-user/.ssh/authorized_keys
# vi /home/dc2-user/.ssh/authorized_keys
# 添加公钥...
# quit

四、nbd挂载方式修改镜像（qemu-nbd）

1、确保已安装nbd模块，加载模块

1
2

# modinfo nbd
# insmod /lib/modules/3.10.0-693.el7.x86_64/kernel/drivers/block/nbd.ko max_part=8

3、建立nbd连接，挂载到目录

1
2
3

# qemu-nbd -c /dev/nbd0 ./CentOS-7-x86_64-GenericCloud-1708-20180123.qcow2
# mkdir -p /mnt/qcows/qcow0
# mount /dev/nbd0p1  /mnt/qcows/qcow0

4、执行chroot

1

# chroot /mnt/qcows/qcow0/

5、执行修改，比如

1
2

$ passwd dc2-user
# 或其它操作

6、修改完毕后解除挂载点，解除连接

1
2

# umount /mnt/qcows/qcow0
# qemu-nbd -d /dev/nbd0p1

五、通过virt-manager挂载虚拟机

1、执行

1

virt-manager

2、新建虚拟机
选择“导入现有磁盘”，“使用ISO镜像”，选择qcow2文件…
如果报：“WARNING : KVM不可用.这可能是因为没有安装KVM软件包,或者没有载入KVM内核模块.您的虚拟机可能性很差。”的警告相应的解决方案是：

  • 关闭虚拟机
  • 进入虚拟机设置（可以配置网卡，硬盘，光驱的地方）
  • 点击“处理器和内存”，勾选虚拟化Inter VT-x/EPT 或AMD-V/RVI(V)

3、登录机器，修改
IP地址和密码、公钥方式登录，在前面已说明如何操作

六、清理痕迹

1、清理/var/log/文件夹
2、删除cloud-init执行记录
cloud-init是专门为云环境的虚拟机初始化而开发的工具，通过读取相应的数据，对虚拟机进行配置。其执行完毕后会在一个叫 sem 的目录下创建信号文件，以便下次启动模块时不再重复执行语句。其目录位置为：
/var/lib/cloud/instances/实例ID/sem

1
2

sudo rm -rf /var/lib/cloud/instances
sudo rm -rf /var/lib/cloud/instance

3、清理history和.ssh目录等

1
2
3
4
5

rm -rf /home/xxx/.ssh
echo '' # /home/xxx/.bash_history
echo '' # /root/.bash_history
 
rm -rf /root/.oracle_jre_usage

七、去除磁盘空洞

1
2
3
4
5
6

# 创建同样大小的镜像
$ qemu-img create -f qcow2 CentOS-7-x86_64-GenericCloud-1708-20180329.qcow2 40G
$ virt-sparsify -x ./CentOS-7-x86_64-GenericCloud-1708-20180123.qcow2 --convert qcow2 ./CentOS-7-x86_64-GenericCloud-1708-20180329.qcow2
$ du -sh *
7.3G	CentOS-7-x86_64-GenericCloud-1708-20180123.qcow2
5.3G	CentOS-7-x86_64-GenericCloud-1708-20180329.qcow2
Permalink | Comment (0)
未分类

Ambari的Stack配置合并

2018-01-16 – 00:33

目的

Ambari的stack有继承机制，一个Service可以从stack或者从common-service中继承，当一个服务从另一个版本的服务继承时，有几种规则，对于不同类型的文件，有不同的规则。不幸的是要研究ambari源码或对实现进行定制修复问题，就需要知道某一个版本的的服务完整的配置是怎样的，而这些继承的规则又非常的复杂，通过手工去合并配置和脚本，工作量很大而且无法保证手动合并的代码正确，几乎是不可能手动去完成。需要有一种机制，既能保证合并效率又能保证合并后配置和脚本的正确性。

实现

Ambari代码会处理继承，只需要把Ambari解析到内存中的StackInfo的对象的数据，写入相应的文件和目录中，就可以简单的实现配置和脚本的合并，下面是实现的配置合并的代码，要运行下面的代码，首先需要生成StackInfo的对象info。

1. 获取StackInfo对象

1
2
3
4
5
6
7

StackManager stackManager = new StackManager(
      new File(stackRoot),
      new File(commonServices), // commonServices设置为null时不会从common-service继承实现
      null,
      osFamily, false,
      metaInfoDao, actionMetadata, stackDao, extensionDao, linkDao, helper);
StackInfo info = stackManager.getStack("HDP", "2.6");

2. 主要实现代码

首先使用ambari内部映射对象去encode文件内容写到文件中，如果出现异常（异常应该不难处理，javax.xml.bind.JAXBException: class java.util.LinkedHashMap nor any of its super class is known to this context　此处没有解决此异常，使用其它方法生成文件）使用模板处理文件内容生成

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234

{
    final File stackDirFile = new File("/tmp/" + info.getName() + "/" + info.getVersion());
    stackDirFile.mkdirs();
 
    StackMetainfoXml stackMetainfoXml = new StackMetainfoXml();
    stackMetainfoXml.setValid(info.isValid());
//    stackMetainfoXml.setExtendsVersion(info.getParentStackVersion());
//    stackMetainfoXml.setMinJdk(info.getMinJdk());
//    stackMetainfoXml.setMaxJdk(info.getMaxJdk());
//    stackMetainfoXml.setVersion(info.getVersion());
    //
    {
      {
        String kerberosDescriptorFileLocation = info.getKerberosDescriptorFileLocation();
        if (kerberosDescriptorFileLocation != null) {
          File file = new File(kerberosDescriptorFileLocation);
          Files.copy(file, new File(stackDirFile.getAbsolutePath(), FilenameUtils.getName(file.getAbsolutePath())));
        }
      }
      {
        String kerberosDescriptorPreConfigurationFileLocation = info.getKerberosDescriptorPreConfigurationFileLocation();
        if (kerberosDescriptorPreConfigurationFileLocation != null) {
          File file = new File(kerberosDescriptorPreConfigurationFileLocation);
          Files.copy(file, new File(stackDirFile.getAbsolutePath(), FilenameUtils.getName(file.getAbsolutePath())));
        }
      }
      {
        String widgetsDescriptorFileLocation = info.getWidgetsDescriptorFileLocation();
        if (widgetsDescriptorFileLocation != null) {
          File file = new File(widgetsDescriptorFileLocation);
          Files.copy(file, new File(stackDirFile.getAbsolutePath(), FilenameUtils.getName(file.getAbsolutePath())));
        }
      }
      {
        StackRoleCommandOrder roleCommandOrder = info.getRoleCommandOrder();
        String value = new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(roleCommandOrder.getContent());
        Files.write(value, new File(stackDirFile.getAbsolutePath(), "role_command_order.json"), Charset.forName("UTF-8"));
      }
    }
 
    // service
    Collection serviceInfos = info.getServices();
    for (ServiceInfo serviceInfo : serviceInfos) {
      String name = serviceInfo.getName();
      File serviceDir = new File(stackDirFile.getAbsolutePath() + "/" + "services", name);
      serviceDir.mkdirs();
 
      {
        File kerberosDescriptorFile = serviceInfo.getKerberosDescriptorFile();
        File widgetsDescriptorFile = serviceInfo.getWidgetsDescriptorFile();
        File metricsFile = serviceInfo.getMetricsFile();
 
        if (kerberosDescriptorFile != null) {
          Files.copy(kerberosDescriptorFile, new File(serviceDir.getAbsolutePath(), FilenameUtils.getName(kerberosDescriptorFile.getAbsolutePath())));
        }
        if (widgetsDescriptorFile != null) {
          Files.copy(widgetsDescriptorFile, new File(serviceDir.getAbsolutePath(), serviceInfo.getWidgetsFileName()));
        }
        if (metricsFile != null) {
          Files.copy(metricsFile, new File(serviceDir.getAbsolutePath(), serviceInfo.getMetricsFileName()));
        }
        {
          StackRoleCommandOrder roleCommandOrder = info.getRoleCommandOrder();
          String value = new ObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsString(roleCommandOrder.getContent());
          Files.write(value, new File(serviceDir.getAbsolutePath(), "role_command_order.json"), Charset.forName("UTF-8"));
        }
      }
 
      if (serviceInfo.getQuickLinksConfigurationsMap() != null &amp;&amp; !serviceInfo.getQuickLinksConfigurationsMap().isEmpty()) {
        File dir = new File(serviceDir.getAbsolutePath(), serviceInfo.getQuickLinksConfigurationsDir());
        dir.mkdirs();
 
        List infos = Lists.newArrayList();
        for (Map.Entry<String, QuickLinksConfigurationInfo> entry : serviceInfo.getQuickLinksConfigurationsMap().entrySet()) {
          infos.add(entry.getValue());
 
          ObjectMapper objectMapper = new ObjectMapper();
          ObjectWriter writer = objectMapper.writerWithDefaultPrettyPrinter();
 
          QuickLinksConfigurationInfo configurationInfo = entry.getValue();
          QuickLinks quickLinks = configurationInfo.getQuickLinksConfigurationMap().values().iterator().next();
          String value = writer.writeValueAsString(quickLinks);
          Files.write(value, new File(dir.getAbsolutePath(), entry.getKey()), Charset.forName("UTF-8"));
        }
        serviceInfo.setQuickLinksConfigurations(infos);
      }
 
      if (serviceInfo.getThemesMap() != null &amp;&amp; !serviceInfo.getThemesMap().isEmpty()) {
        File themeDir = new File(serviceDir.getAbsolutePath(), serviceInfo.getThemesDir());
        themeDir.mkdirs();
 
        List themes = Lists.newArrayList();
        for (Map.Entry<String, ThemeInfo> entry : serviceInfo.getThemesMap().entrySet()) {
          themes.add(entry.getValue());
 
          ObjectMapper objectMapper = new ObjectMapper();
          ObjectWriter writer = objectMapper.writerWithDefaultPrettyPrinter();
 
          ThemeInfo themeInfo = entry.getValue();
          Theme theme = themeInfo.getThemeMap().values().iterator().next();
          String value = writer.writeValueAsString(theme);
          Files.write(value, new File(themeDir.getAbsolutePath(), entry.getKey()), Charset.forName("UTF-8"));
        }
        serviceInfo.setThemes(themes);
      }
 
      // 生成configuration配置文件
      List properties = serviceInfo.getProperties();
      Set pFileNames = Sets.newLinkedHashSet();
      Set pFileSortNames = Sets.newLinkedHashSet();
      Map<String, List> configurationFiles = Maps.newLinkedHashMap();
      // configuration
      for (PropertyInfo propertyInfo : properties) {
        String filename = propertyInfo.getFilename();
        pFileNames.add(filename);
        pFileSortNames.add(FilenameUtils.getBaseName(filename));
 
        List propertyInfos = configurationFiles.get(filename);
        if (propertyInfos == null) {
          configurationFiles.put(filename, new ArrayList());
          propertyInfos = configurationFiles.get(filename);
        }
        propertyInfos.add(propertyInfo);
      }
      if (configurationFiles.size() > 0) {
        File configDir = new File(serviceDir.getAbsolutePath() + "/configuration");
        configDir.mkdirs();
 
        for (Map.Entry<String, List> entry : configurationFiles.entrySet()) {
          String fileName = entry.getKey();
          String shortName = FilenameUtils.getBaseName(fileName);
          {// configuration
            ConfigurationXml configurationXml = new ConfigurationXml();
            configurationXml.addAll(entry.getValue());
            Map<QName, String> attri = Maps.newLinkedHashMap();
//            attri.put(new QName("configuration.xsl", "text/xsl", "kkkk"), "yyyy");
            // configuration xmlns:kkkk="configuration.xsl" kkkk:text/xsl="yyyy">
            Map<String, String> supports = serviceInfo.getConfigTypeAttributes().get(shortName).get("supports");
 
            attri.put(new QName(null, "supports_adding_forbidden", ""), supports.get("adding_forbidden"));
            attri.put(new QName(null, "supports_final", ""), supports.get("final"));
            attri.put(new QName(null, "supports_do_not_extend", ""), supports.get("do_not_extend"));
            configurationXml.addAll(attri);
            FileOutputStream stream = null;
            try {
              stream = new FileOutputStream(new File(configDir.getAbsolutePath(), fileName));
              JAXBContext jc = JAXBContext.newInstance(ConfigurationXml.class);
              Marshaller ms = jc.createMarshaller();
              ms.setProperty(Marshaller.JAXB_ENCODING, "UTF-8");//编码格式
              ms.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
              ms.setProperty(Marshaller.JAXB_FRAGMENT, false);
              ms.setProperty("com.sun.xml.bind.xmlHeaders", "\n<!--?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?-->\n" +
                "<!--\n" + "/**\n" + " * Licensed to the Apache Software Foundation (ASF) under one\n" + " * or more contributor license agreements. See the NOTICE file\n" + " * distributed with this work for additional information\n" + " * regarding copyright ownership. The ASF licenses this file\n" + " * to you under the Apache License, Version 2.0 (the\n" + " * \"License\"); you may not use this file except in compliance\n" + " * with the License. You may obtain a copy of the License at\n" + " *\n" + " * http://www.apache.org/licenses/LICENSE-2.0\n" + " *\n" + " * Unless required by applicable law or agreed to in writing, software\n" + " * distributed under the License is distributed on an \"AS IS\" BASIS,\n" + " * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + " * See the License for the specific language governing permissions and\n" + " * limitations under the License.\n" + " */\n" + "-->\n");
 
              ms.marshal(configurationXml, stream);
            } catch (JAXBException e) {
//        e.printStackTrace();
              new File(stackDirFile.getAbsolutePath() + "/" + "configuration_0").mkdirs();
              FileOutputStream outputStream = new FileOutputStream(new File(stackDirFile.getAbsolutePath() + "/" + "configuration_0", name));
              e.printStackTrace(new PrintStream(outputStream, true));
              outputStream.close();
            } finally {
              Closeables.close(stream, true);
            }
          }
//          {
//            // configuration Deprecate
//            VelocityContext context = new VelocityContext();
//            List propertyInfos = entry.getValue();
//            propertyInfos = Lists.transform(propertyInfos, new Function<PropertyInfo, PropertyInfo>() {
//              public PropertyInfo apply(PropertyInfo input) {
//                String value = input.getValue();
//                if (value != null &amp;&amp; (value.contains("<") || value.contains(">") || value.contains("/") || value.length() > 1024)) {
//                  input.setValue(String.format("<![CDATA[%s]]>", value));
//                }
//                return input;
//              }
//            });
//            context.put("properties", propertyInfos);
//            context.put("configType", serviceInfo.getConfigTypeAttributes().get(shortName));
//            Template template = VelocityUtil.getTemplateByName("configuration");
//            FileWriter writer = new FileWriter(new File(serviceDir.getAbsolutePath() + "/configuration", fileName));
//            template.merge(context, writer);
//            writer.flush();
//            writer.close();
//          }
        }
      }
//      FileWriter fileWriter = new FileWriter("/tmp/HDP/server_" + serviceInfo.getName());
//      GsonBuilder gsonBuilder = new GsonBuilder().setPrettyPrinting();
//      fileWriter.write(gsonBuilder.create().toJson(serviceInfo));
//      fileWriter.close();
      // 生成metainfo.xml
      FileOutputStream stream = null;
      try {
        JAXBContext jc = JAXBContext.newInstance(ServiceMetainfoXml.class);
        Marshaller ms = jc.createMarshaller();
        ms.setProperty(Marshaller.JAXB_ENCODING, "UTF-8");//编码格式
        ms.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true);
        ms.setProperty(Marshaller.JAXB_FRAGMENT, false);
        ms.setProperty("com.sun.xml.bind.xmlHeaders", "\n\n" +
          "<!--\n" + " Licensed to the Apache Software Foundation (ASF) under one or more\n" + " contributor license agreements. See the NOTICE file distributed with\n" + " this work for additional information regarding copyright ownership.\n" + " The ASF licenses this file to You under the Apache License, Version 2.0\n" + " (the \"License\"); you may not use this file except in compliance with\n" + " the License. You may obtain a copy of the License at\n" + "\n" + " http://www.apache.org/licenses/LICENSE-2.0\n" + "\n" + " Unless required by applicable law or agreed to in writing, software\n" + " distributed under the License is distributed on an \"AS IS\" BASIS,\n" + " WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + " See the License for the specific language governing permissions and\n" + " limitations under the License.\n" + "-->\n");
 
        ServiceMetainfoXml serviceMetainfoXml = new ServiceMetainfoXml();
        serviceMetainfoXml.setServices(Lists.newArrayList(serviceInfo));
        serviceMetainfoXml.setSchemaVersion("2.0");
        serviceMetainfoXml.setValid(serviceInfo.isValid());
 
        stream = new FileOutputStream(new File(serviceDir.getAbsolutePath(), "metainfo.xml"));
        ms.marshal(serviceMetainfoXml, stream);
      } catch (JAXBException e) {
//        e.printStackTrace();
        new File(stackDirFile.getAbsolutePath() + "/" + "services_2").mkdirs();
        FileOutputStream outputStream = new FileOutputStream(new File(stackDirFile.getAbsolutePath() + "/" + "services_2", name));
        e.printStackTrace(new PrintStream(outputStream, true));
        outputStream.close();
 
        {// @Deprecate
          // meteinfo.xml
          VelocityContext context = new VelocityContext();
          context.put("serviceInfo", serviceInfo);
          context.put("pFileNames", pFileNames);
          context.put("pFileSortNames", pFileSortNames);
          Template template = VelocityUtil.getTemplateByName("metainfo");
          FileWriter writer = new FileWriter(new File(serviceDir.getAbsolutePath(), "metainfo.xml"));
          template.merge(context, writer);
          writer.flush();
          writer.close();
        }
      } finally {
        Closeables.close(stream, true);
      }
    }
  }

Velocity模板类

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

public final class VelocityUtil {
  private static final Logger LOGGER = LoggerFactory.getLogger(VelocityUtil.class);
 
  private VelocityUtil() {/**/}
 
  private static final VelocityEngine VELOCITY_ENGINE;
 
  static {
    VELOCITY_ENGINE = new VelocityEngine();
    VELOCITY_ENGINE.setProperty(RuntimeConstants.RESOURCE_LOADER, "classpath");
    VELOCITY_ENGINE.setProperty("userdirective", "org.apache.ambari.server.utils.Ifnull");
    VELOCITY_ENGINE.setProperty("userdirective", "org.apache.ambari.server.utils.Ifnotnull");
    VELOCITY_ENGINE.setProperty("classpath.resource.loader.class", ClasspathResourceLoader.class.getName());
    VELOCITY_ENGINE.init();
  }
 
  public static Template getTemplateByName(String templateName) {
    return VELOCITY_ENGINE.getTemplate("velocity/" + templateName + ".vm", "utf8");
  }
 
  public static String getContentByTemplate(String templateName, Map<String, Object> datas) {
    try {
      Template template = VELOCITY_ENGINE.getTemplate("velocity/" + templateName + ".vm", "utf8");
      VelocityContext context = new VelocityContext();
      for (Map.Entry<String, Object> data : datas.entrySet()) {
        context.put(data.getKey(), data.getValue());
      }
      StringWriter writer = new StringWriter();
      template.merge(context, writer);
      return writer.toString();
    } catch (Exception e) {
      LOGGER.error("对应模板文件未找到", e);
    }
    return null;
  }
 
}

模板 configuration.vm

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107

<!--?xml version="1.0"?-->
<!--?xml-stylesheet type="text/xsl" href="configuration.xsl"?-->
<!-- /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -->
#macro(output $__p_page, $box)
    #ifnotnull($__p_page)
        #if($__p_page != "")
            <$!{box}>$__p_page
        #end
    #end
#end
 
 
#foreach($entry in $properties)
 
        $!{entry.name}
        $!{entry.value}
        $!{entry.description}
        $!{entry.displayName}
        $!{entry.filename}
        $!{entry.deleted}
        $!{entry.deleted}
 
        #ifnotnull($entry.propertyAmbariUpgradeBehavior)
 
        #end
        #if($entry.propertyTypes.size()>0)
            #foreach($ept in $entry.propertyTypes)$!{ept}#end
        #end
        #ifnotnull($entry.propertyValueAttributes)
 
            #output("$!{entry.propertyValueAttributes.type}", "type")
            #output("$!{entry.propertyValueAttributes.minimum}", "minimum")
            #output("$!{entry.propertyValueAttributes.maximum}", "maximum")
            #output("$!{entry.propertyValueAttributes.unit}", "unit")
            #output("$!{entry.propertyValueAttributes.delete}", "delete")
            #output("$!{entry.propertyValueAttributes.visible}", "visible")
            #output("$!{entry.propertyValueAttributes.overridable}", "overridable")
            #output("$!{entry.propertyValueAttributes.copy}", "copy")
            #output("$!{entry.propertyValueAttributes.emptyValueValid}", "empty-value-valid")
            #output("$!{entry.propertyValueAttributes.uiOnlyProperty}", "ui-only-property")
            #output("$!{entry.propertyValueAttributes.readOnly}", "readOnly")
            #output("$!{entry.propertyValueAttributes.editableOnlyAtInstall}", "editable-only-at-install")
            #output("$!{entry.propertyValueAttributes.showPropertyName}", "show-property-name")
            #output("$!{entry.propertyValueAttributes.incrementStep}", "increment-step")
            #output("$!{entry.propertyValueAttributes.keyStore}", "keyStore")
 
            #if($entry.propertyValueAttributes.entries.size()>0)
 
                #foreach($pvae in $entry.propertyValueAttributes.entries)
 
                        #output("$!{pvae.value}", "value")
                        #output("$!{pvae.label}", "label")
                        #output("$!{pvae.description}", "description")
 
                #end
 
            #end
            #output("$!{entry.propertyValueAttributes.hidden}", "hidden")
            #output("$!{entry.propertyValueAttributes.entriesEditable}", "entries_editable")
            #output("$!{entry.propertyValueAttributes.selectionCardinality}", "selection-cardinality")
            #output("$!{entry.propertyValueAttributes.propertyFileName}", "property-file-name")
            #output("$!{entry.propertyValueAttributes.propertyFileType}", "property-file-type")
            #output("$!{entry.propertyValueAttributes.propertyFileType}", "property-file-type")
            #if($entry.propertyValueAttributes.userGroupEntries.size()>0)
 
                #foreach($pvae in $entry.propertyValueAttributes.userGroupEntries)
 
                        #output("$!{pvae.value}", "value")
                        #output("$!{pvae.label}", "label")
 
                #end
 
            #end
 
        #end
        #if($entry.dependsOnProperties.size()>0)
 
            #foreach($edop in $entry.dependsOnProperties)
 
                    #output("$!{pvae.name}", "name")
                    #output("$!{pvae.type}", "type")
 
            #end
 
        #end
        #if($entry.dependedByProperties.size()>0)
 
            #foreach($edop in $entry.dependedByProperties)
 
                    #output("$!{pvae.name}", "name")
                    #output("$!{pvae.type}", "type")
 
            #end
 
        #end
        #if($entry.usedByProperties.size()>0)
 
            #foreach($edop in $entry.usedByProperties)
 
                    #output("$!{pvae.name}", "name")
                    #output("$!{pvae.type}", "type")
 
            #end
 
        #end
 
#end

模板 metainfo.vm

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225

<!--?xml version="1.0"?-->
<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -->
#macro(output $__p_page, $box)
    #ifnotnull($__p_page)
        #if($__p_page != "")
        <$!{box}>$__p_page
        #end
    #end
#end
#set($service = "service")
#set($services = "services")
 
    $!{serviceInfo.schemaVersion}
    <${services}>
        <${service}>
            #output("$!{serviceInfo.name}", "name")
            #output("$!{serviceInfo.displayName}", "displayName")
            #output("$!{serviceInfo.version}", "version")
            #output("$!{serviceInfo.comment}", "comment")
            #output("$!{serviceInfo.serviceType}", "serviceType")
            #output("$!{serviceInfo.selection}", "selection")
 
            #ifnotnull($serviceInfo.credentialStoreInfo)
 
                #output("$!{serviceInfo.credentialStoreInfo.supported}", "supported")
                #output("$!{serviceInfo.credentialStoreInfo.enabled}", "enabled")
                #output("$!{serviceInfo.credentialStoreInfo.required}", "required")
 
            #end
 
            #if($serviceInfo.components.size()>0)
 
                    #foreach($entry in $serviceInfo.components)
 
                            #output("$!{entry.name}", "name")
                            #output("$!{entry.displayName}", "displayName")
                            #output("$!{entry.category}", "category")
                            #output("$!{entry.deleted}", "deleted")
                            #output("$!{entry.cardinality}", "cardinality")
                            #output("$!{entry.versionAdvertisedField}", "versionAdvertised")
                            #output("$!{entry.versionAdvertisedInternal}", "versionAdvertisedInternal")
                            #output("$!{entry.unlimitedKeyJCERequired}", "unlimitedKeyJCERequired")
                            #output("$!{entry.rollingRestartSupported}", "rollingRestartSupported")
 
                            #ifnotnull($entry.commandScript)
 
                                    #output("$!{entry.commandScript.script}", "script")
                                    #output("$!{entry.commandScript.scriptType}", "scriptType")
                                    #output("$!{entry.commandScript.timeout}", "timeout")
 
                            #end
                            #if($entry.logs.size()>0)
 
                                    #foreach($log in $entry.logs)
 
                                            #output("$!{log.logId}", "logId")
                                            #output("$!{log.primary}", "primary")
 
                                    #end
 
                            #end
                            #if($entry.clientsToUpdateConfigs.size()>0)
 
                                    #foreach($client in $entry.clientsToUpdateConfigs)
                                        #output("$!{client}", "client")
                                    #end
 
                            #end
                            #if($entry.configFiles.size()>0)
 
                                    #foreach($configFile in $entry.configFiles)
 
                                            #output("$!{configFile.type}", "type")
                                            #output("$!{configFile.fileName}", "fileName")
                                            #output("$!{configFile.dictionaryName}", "dictionaryName")
                                            #output("$!{configFile.optional}", "optional")
 
                                    #end
 
                            #end
                            #if($entry.customCommands.size()>0)
 
                                    #foreach($customCommand in $entry.customCommands)
 
                                            #output("$!{customCommand.name}", "name")
                                            #output("$!{customCommand.background}", "background")
 
                                            #ifnotnull($customCommand.commandScript)
 
                                                #output("$!{customCommand.commandScript.script}", "script")
                                                #output("$!{customCommand.commandScript.scriptType}", "scriptType")
                                                #output("$!{customCommand.commandScript.timeout}", "timeout")
 
                                            #end
 
                                    #end
 
                            #end
 
                            #ifnotnull($entry.bulkCommandDefinition)
 
                                    #output("$!{entry.bulkCommandDefinition.displayName}", "displayName")
                                    #output("$!{entry.bulkCommandDefinition.masterComponent}", "masterComponent")
 
                            #end
 
                            #if($entry.dependencies.size()>0)
 
                                    #foreach($dependency in $entry.dependencies)
                                        #output("$!{dependency.name}", "name")
                                        #output("$!{dependency.scope}", "scope")
                                        #output("$!{dependency.serviceName}", "serviceName")
                                        #output("$!{dependency.componentName}", "componentName")
                                        #ifnotnull($dependency.autoDeploy)
 
                                            #output("$!{dependency.autoDeploy.m_enabled}", "enabled")
                                            #output("$!{dependency.autoDeploy.m_coLocate}", "co-locate")
 
                                        #end
                                    #end
 
                            #end
                            #if($entry.configDependencies.size()>0)
 
                                    #foreach($dependency in $entry.configDependencies)
                                        #output("$!{dependency}", "config-type")
                                    #end
 
                            #end
 
                            #ifnotnull($entry.autoDeploy)
 
                                #output("$!{entry.autoDeploy.m_enabled}", "extended")
                                #output("$!{entry.autoDeploy.m_coLocate}", "co-locate")
 
                            #end
 
                            #output("$!{entry.recoveryEnabled}", "recovery_enabled")
                            #output("$!{entry.reassignAllowed}", "reassignAllowed")
                            #output("$!{entry.timelineAppid}", "timelineAppid")
                            #output("$!{entry.customFolder}", "customFolder")
 
                    #end
 
            #end
            #output("$!{serviceInfo.parent}", "extends")
            #output("$!{serviceInfo.widgetsFileName}", "widgetsFileName")
            #output("$!{serviceInfo.metricsFileName}", "metricsFileName")
 
            #output("$!{serviceInfo.isDeleted}", "deleted")
 
            #if($serviceInfo.configLayout.size()>0)
 
                    #foreach($layout in $serviceInfo.configLayout.entrySet())
                    <$!{layout.key}>
                        $!{layout.value}
 
                    #end
 
            #end
            #if($serviceInfo.configDependencies.size()>0)
 
                    #foreach($depend in $serviceInfo.configDependencies)
                        #output("$!{depend}", "config-type")
                    #end
 
            #end
            #if($serviceInfo.excludedConfigTypes.size()>0)
 
                    #foreach($depend in $serviceInfo.excludedConfigTypes)
                        #output("$!{depend}", "config-type")
                    #end
 
            #end
            #if($serviceInfo.configTypes.size()>0)
 
                    #foreach($ctype in $serviceInfo.configTypes.entrySet())
                        <$!{ctype.key}>
                            #foreach($cv in $ctype.value.entrySet())
                                <$!{cv.key}>$!{cv.value}
                            #end
 
                    #end
 
            #end
 
##            #if($pFileSortNames.size()>0) // TODO: xxx
##                
##                    #foreach($name in $pFileSortNames)
##                        #output("$!{name}", "config-type")
##                    #end
##                
##            #end
##
            #if($serviceInfo.requiredServices.size()>0)
 
                #foreach($name in $serviceInfo.requiredServices)
                    #output("$!{name}", "service")
                #end
 
            #end
##
            #if($serviceInfo.themesMap.size()>0)
 
                #foreach($entry in $serviceInfo.themesMap.entrySet())
 
                        #output("$!{entry.key}", "fileName")
                        #output("$!{entry.value.isDefault}", "default")
                        #output("$!{entry.value.deleted}", "deleted")
 
                #end
 
            #end
 
    #if($serviceInfo.themesMap.size()>0)
 
            #foreach($entry in $serviceInfo.themesMap.entrySet())
 
                    #output("$!{entry.key}", "fileName")
                        #output("$!{entry.value.isDefault}", "default")
                        #output("$!{entry.value.deleted}", "deleted")
 
            #end
 
    #end
Permalink | Comment (0)

Next Page »

Next Page »