1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
| [root@b94a78ebfc80 ~]$ gdb glusterd
GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-115.el7
Copyright (C) 2013 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-redhat-linux-gnu".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from /usr/local/sbin/glusterfsd...done.
(gdb) br glusterfsd.c:main
Breakpoint 1 at 0x40b71d: file glusterfsd.c, line 2746.
(gdb) br glusterd.c:init
No source file named glusterd.c.
Make breakpoint pending on future shared library load? (y or [n]) y
Breakpoint 2 (glusterd.c:init) pending.
(gdb) set print pretty on
(gdb) set follow-fork-mode child
(gdb) set detach-on-fork off
(gdb) c
The program is not being run.
(gdb) r
Starting program: /usr/local/sbin/glusterd
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Breakpoint 1, main (argc=1, argv=0x7fffffffe778) at glusterfsd.c:2746
2746 glusterfs_ctx_t *ctx = NULL;
Missing separate debuginfos, use: debuginfo-install glibc-2.17-292.el7.x86_64 libuuid-2.23.2-61.el7_7.1.x86_64 openssl-libs-1.0.2k-19.el7.x86_64 zlib-1.2.7-18.el7.x86_64
(gdb) c
Continuing.
[New process 956]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
[New Thread 0x7ffff5431700 (LWP 957)]
[New Thread 0x7ffff4c30700 (LWP 958)]
[New Thread 0x7ffff442f700 (LWP 959)]
[New Thread 0x7ffff3c2e700 (LWP 960)]
[New Thread 0x7ffff342d700 (LWP 961)]
[New Thread 0x7ffff2c2c700 (LWP 962)]
Reading symbols from /usr/local/lib/glusterfs//xlator/mgmt/glusterd.so...done.
[Switching to Thread 0x7ffff7fea4c0 (LWP 956)]
Breakpoint 2, init (this=0x686d40) at glusterd.c:1373
1373 int32_t ret = -1;
Missing separate debuginfos, use: debuginfo-install glibc-2.17-292.el7.x86_64 libuuid-2.23.2-61.el7_7.1.x86_64 libxml2-2.9.1-6.el7_2.3.x86_64 openssl-libs-1.0.2k-19.el7.x86_64 userspace-rcu-0.7.16-1.el7.x86_64 xz-libs-5.2.2-1.el7.x86_64 zlib-1.2.7-18.el7.x86_64
(gdb) set follow-fork-mode parent
(gdb) set detach-on-fork on
(gdb) br glusterd_spawn_daemons
Breakpoint 3 at 0x7ffff1f79f85: file glusterd-utils.c, line 3619.
(gdb) br glusterd_restart_bricks
Breakpoint 4 at 0x7ffff1f832e3: file glusterd-utils.c, line 6334.
(gdb) br glusterd_brick_start
Breakpoint 5 at 0x7ffff1f826cd: file glusterd-utils.c, line 6107.
(gdb) br glusterd_volume_start_glusterfs
Breakpoint 6 at 0x7ffff1f73f5f: file glusterd-utils.c, line 2024.
(gdb) c
Continuing.
Reading symbols from /usr/local/lib/glusterfs//rpc-transport/socket.so...done.
Reading symbols from /usr/local/lib/glusterfs//rpc-transport/rdma.so...done.
Detaching after fork from child process 964.
Detaching after fork from child process 965.
Detaching after fork from child process 966.
Detaching after fork from child process 967.
Detaching after fork from child process 968.
Detaching after fork from child process 969.
Detaching after fork from child process 970.
Detaching after fork from child process 971.
Detaching after fork from child process 972.
Detaching after fork from child process 973.
Detaching after fork from child process 974.
Detaching after fork from child process 975.
Detaching after fork from child process 976.
Detaching after fork from child process 977.
Detaching after fork from child process 978.
Detaching after fork from child process 979.
Detaching after fork from child process 980.
Detaching after fork from child process 981.
Detaching after fork from child process 982.
Detaching after fork from child process 983.
Detaching after fork from child process 984.
Detaching after fork from child process 985.
Detaching after fork from child process 986.
Detaching after fork from child process 987.
Detaching after fork from child process 988.
[Switching to Thread 0x7ffff342d700 (LWP 961)]
Breakpoint 3, glusterd_spawn_daemons (opaque=0x0) at glusterd-utils.c:3619
3619 glusterd_conf_t *conf = THIS->private;
Missing separate debuginfos, use: debuginfo-install keyutils-libs-1.5.8-3.el7.x86_64 krb5-libs-1.15.1-37.el7_7.2.x86_64 libcom_err-1.42.9-16.el7.x86_64 libibverbs-22.1-3.el7.x86_64 libnl3-3.2.28-4.el7.x86_64 librdmacm-22.1-3.el7.x86_64 libselinux-2.5-14.1.el7.x86_64 pcre-8.32-17.el7.x86_64
(gdb) n
3620 int ret = -1;
(gdb)
3622 synclock_lock(&conf->big_lock);
(gdb)
[New Thread 0x7fffef606700 (LWP 989)]
3623 glusterd_restart_bricks();
(gdb)
Breakpoint 4, glusterd_restart_bricks (opaque=0x7fffef9a9fc8) at glusterd-utils.c:6334
6334 int ret = 0;
(gdb)
6335 glusterd_volinfo_t *volinfo = NULL;
(gdb)
6336 glusterd_brickinfo_t *brickinfo = NULL;
(gdb) set follow-fork-mode child
(gdb) set detach-on-fork on
(gdb) br runner_run
Breakpoint 7 at 0x7ffff7b35f04: runner_run. (2 locations)
(gdb) br runner_start
Breakpoint 8 at 0x7ffff7b35825: runner_start. (2 locations)
(gdb) info break
Num Type Disp Enb Address What
1 breakpoint keep y <MULTIPLE>
breakpoint already hit 1 time
1.1 y 0x000000000040b71d in main at glusterfsd.c:2746 inf 1
1.2 y 0x000000000040b71d in main at glusterfsd.c:2746 inf 2
2 breakpoint keep y 0x00007ffff1f3a7ee in init at glusterd.c:1373 inf 2
breakpoint already hit 1 time
3 breakpoint keep y 0x00007ffff1f79f85 in glusterd_spawn_daemons at glusterd-utils.c:3619 inf 2
breakpoint already hit 1 time
4 breakpoint keep y 0x00007ffff1f832e3 in glusterd_restart_bricks at glusterd-utils.c:6334 inf 2
breakpoint already hit 1 time
5 breakpoint keep y 0x00007ffff1f826cd in glusterd_brick_start at glusterd-utils.c:6107 inf 2
6 breakpoint keep y 0x00007ffff1f73f5f in glusterd_volume_start_glusterfs at glusterd-utils.c:2024 inf 2
7 breakpoint keep y <MULTIPLE>
7.1 y 0x00007ffff7b35f04 in runner_run at run.c:430 inf 1
7.2 y 0x00007ffff7b35f04 in runner_run at run.c:430 inf 2
8 breakpoint keep y <MULTIPLE>
8.1 y 0x00007ffff7b35825 in runner_start at run.c:259 inf 1
8.2 y 0x00007ffff7b35825 in runner_start at run.c:259 inf 2
(gdb) c
Continuing.
Breakpoint 5, glusterd_brick_start (volinfo=0x6db4b0, brickinfo=0x6ef490, wait=false, only_connect=false) at glusterd-utils.c:6107
6107 int ret = -1;
(gdb) c
Continuing.
[New Thread 0x7fffeee05700 (LWP 991)]
Breakpoint 6, glusterd_volume_start_glusterfs (volinfo=0x6db4b0, brickinfo=0x6ef490, wait=false) at glusterd-utils.c:2024
2024 int32_t ret = -1;
(gdb) c
Continuing.
[New process 992]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
[Switching to Thread 0x7ffff342d700 (LWP 992)]
Breakpoint 8, runner_start (runner=0x7fffef800e00) at run.c:259
259 int pi[3][2] = {{-1, -1}, {-1, -1}, {-1, -1}};
(gdb) n
261 int ret = 0;
(gdb)
262 int errno_priv = 0;
(gdb)
263 int i = 0;
(gdb)
266 if (runner->runerr || !runner->argv) {
(gdb)
271 GF_ASSERT(runner->argv[0]);
(gdb) set follow-fork-mode child
(gdb) set detach-on-fork on
(gdb) n
276 ret = pipe(xpi);
(gdb)
277 if (ret != -1)
(gdb)
278 ret = fcntl(xpi[1], F_SETFD, FD_CLOEXEC);
(gdb)
280 for (i = 0; i < 3; i++) {
(gdb)
281 if (runner->chfd[i] != -2)
(gdb)
282 continue;
(gdb)
280 for (i = 0; i < 3; i++) {
(gdb)
281 if (runner->chfd[i] != -2)
(gdb)
282 continue;
(gdb)
280 for (i = 0; i < 3; i++) {
(gdb)
281 if (runner->chfd[i] != -2)
(gdb)
282 continue;
(gdb)
280 for (i = 0; i < 3; i++) {
(gdb)
291 if (ret != -1)
(gdb)
292 runner->chpid = fork();
(gdb)
[New process 995]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
[Switching to Thread 0x7ffff342d700 (LWP 995)]
293 switch (runner->chpid) {
327 if (ret != -1) {
(gdb)
328 int fdv[4] = {0, 1, 2, xpi[1]};
(gdb)
330 ret = close_fds_except(fdv, sizeof(fdv) / sizeof(*fdv));
(gdb)
333 if (ret != -1) {
(gdb)
335 sigemptyset(&set);
(gdb)
336 sigprocmask(SIG_SETMASK, &set, NULL);
(gdb)
338 execvp(runner->argv[0], runner->argv);
(gdb) p runner->argv[0]
$1 = 0x7fffe8000c60 "/usr/local/sbin/glusterfsd"
(gdb) p runner->argv
$2 = (char **) 0x7fffe8000970
(gdb) p/s runner->argv
$3 = (char **) 0x7fffe8000970
(gdb) p/s runner->argv[1]
$4 = 0x7fffe8000ce0 "-s"
(gdb) p/s runner->argv[2]
$5 = 0x7fffe8000d50 "172.17.0.2"
(gdb) p/s runner->argv[3]
$6 = 0x7fffe8000dc0 "--volfile-id"
(gdb) p/s runner->argv[4]
$7 = 0x7fffe8000e30 "dht_vol.172.17.0.2.data-brick"
(gdb) p/s runner->argv[5]
$8 = 0x7fffe8000eb0 "-p"
(gdb) p/s runner->argv[6]
$9 = 0x7fffe8000f20 "/var/run/gluster/vols/dht_vol/172.17.0.2-data-brick.pid"
(gdb) p/s runner->argv[7]
$10 = 0x7fffe8000fc0 "-S"
(gdb) p/s runner->argv[8]
$11 = 0x7fffe8001030 "/var/run/gluster/cb954c564295c3be.socket"
(gdb) p/s runner->argv[9]
$12 = 0x7fffe80010c0 "--brick-name"
(gdb) p/s runner->argv[10]
$13 = 0x7fffe8001130 "/data/brick"
(gdb) p/s runner->argv[11]
$14 = 0x7fffe80011a0 "-l"
(gdb) p/s runner->argv[12]
$15 = 0x7fffe8001210 "/var/log/glusterfs/bricks/data-brick.log"
(gdb) p/s runner->argv[13]
$16 = 0x7fffe80012a0 "--xlator-option"
(gdb) p/s runner->argv[14]
$17 = 0x7fffe8001310 "*-posix.glusterd-uuid=88efffbe-ce9e-4117-8d1a-77de0a8ecd75"
(gdb) p/s runner->argv[15]
$18 = 0x7fffe80013b0 "--process-name"
(gdb) p/s runner->argv[16]
$19 = 0x7fffe8001420 "brick"
(gdb) p/s runner->argv[17]
$20 = 0x7fffe8001490 "--brick-port"
(gdb) p/s runner->argv[18]
$21 = 0x7fffe8001500 "49152"
(gdb) p/s runner->argv[19]
$22 = 0x7fffe8001570 "--xlator-option"
(gdb) p/s runner->argv[20]
$23 = 0x7fffe80015e0 "dht_vol-server.listen-port=49152"
(gdb) p/s runner->argv[21]
$24 = 0x0
(gdb)
|
