Zsh Mailing List Archive
Messages sorted by: Reverse Date, Date, Thread, Author

Re: timeout problem in ssh sessions



On 2008-02-12, 08:37, Bart Schaefer wrote:
> If you have or can install "lsof" it would be very interesting to find
> out to what file descriptor 3 is connected -- or more generally, what
> are all the open descriptors in ssh, after the zsh process has exited.

I tried harder this time.  According to the proc fs it's a socket:

condor:/proc/13138/fd#ll
insgesamt 0
lr-x------ 1 root root 64 12. Feb 23:55 0 -> /dev/null
l-wx------ 1 root root 64 12. Feb 23:55 1 -> pipe:[16122]
l-wx------ 1 root root 64 12. Feb 23:55 2 -> pipe:[16122]
lrwx------ 1 root root 64 12. Feb 23:55 3 -> socket:[37169512]
lr-x------ 1 root root 64 12. Feb 23:55 4 -> /dev/null
lrwx------ 1 root root 64 12. Feb 23:55 5 -> /dev/ptmx

The particular process was:
USER       PID  PPID S CPU %MEM   RSS    VSZ  START     ELAPSED   TIME CMD
spiegl   13138     1 S   -  0.1  5332  10024  23:54       03:00   0:00 xterm -title root@condor -geometry 105x45 -e ssh -l root localhost
spiegl   13139 13138 S   -  0.0  2232   4992  23:54       03:00   0:00 ssh -l root localhost
(as you can see, zsh had already terminated)

The strace output of this particular process looked like this:
 ...
 write(3, "\0204\20\16\206\360o\3530=\307,W\221\305\212F\255\250\236"..., 32) = 32
 select(9, [3 8], [], NULL, NULL)        = 1 (in [3])
 read(3, "\304k\346\203T)\220N\222RG\255\177j\331\37\23{\360\323"..., 8192) = 64
 select(9, [3 8], [3], NULL, NULL)       = 1 (out [3])
 write(3, "9Q\320\323:\341;\227s\361\331\315\345\333\336\210ON\375"..., 32) = 32
 select(9, [3 8], [], NULL, NULL)        = 1 (in [8])
 read(8, "\35\17\26\0\320\240\340\17\1\0`\6\276\1\0\0\0\20\0\0\0"..., 16384) = 32
 select(9, [3 8], [3], NULL, NULL)       = 1 (out [3])
 write(3, "3O\0003\362\17\235\267\354\331\35be\337\245\270F\271\203"..., 80) = 80
 select(9, [3 8], [], NULL, NULL)        = 1 (in [3])
 read(3, "9\322\247\16\10\201\265Dus\305J\345\2654\364\342\367\374"..., 8192) = 32
 shutdown(8, 1 /* send */)               = 0
 select(9, [3], [3], NULL, NULL)         = 1 (out [3])
 write(3, ",5\230\22\300\252\341\220\23a\35\315\253Ny\346$\335\3|"..., 64) = 64
 select(9, [3], [], NULL, NULL)          = 1 (in [3])
 read(3, "\267\200_>4\331c\234\215}\31\277\325\264\342R\320p\274"..., 8192) = 32
 shutdown(8, 2 /* send and receive */)   = 0
 close(8)                                = 0
 close(8)                                = -1 EBADF (Bad file descriptor)
 close(8)                                = -1 EBADF (Bad file descriptor)
 rt_sigaction(SIGWINCH, NULL, {0x8000b9c0, [], 0}, 8) = 0
 rt_sigaction(SIGWINCH, {SIG_DFL}, NULL, 8) = 0
 ioctl(0, SNDCTL_TMR_TIMEBASE or TCGETS, {B38400 opost isig icanon echo ...}) = 0
 ioctl(1, SNDCTL_TMR_TIMEBASE or TCGETS, {B38400 opost isig icanon echo ...}) = 0
 ioctl(2, SNDCTL_TMR_TIMEBASE or TCGETS, {B38400 opost isig icanon echo ...}) = 0
 write(2, "Connection to localhost closed.\r"..., 33) = 33
 gettimeofday({1202857222, 359943}, NULL) = 0
 shutdown(3, 2 /* send and receive */)   = 0
 close(3)                                = 0
 brk(0x8007f000)                         = 0x8007f000
 exit_group(0)                           = ?
 Process 13139 detached

After "select(9, [3]..." it was always pausing for several seconds.
Then finally the ssh and xterm window closed.

> It looks from the strace output as if it might be something related to
> the ssh-agent
The ssh-agent had PID 7656 and these file descriptors:

COMMAND     PID       USER   FD      TYPE     DEVICE     SIZE       NODE NAME
ssh-agent  7656     spiegl  cwd       DIR        8,6     4096          2 /
ssh-agent  7656     spiegl  rtd       DIR        8,6     4096          2 /
ssh-agent  7656     spiegl  txt       REG        8,6    77220   15189338 /usr/bin/ssh-agent
ssh-agent  7656     spiegl  mem       REG        8,6            41812070 /lib/libkeyutils-1.2.so (path inode=41812034)
ssh-agent  7656     spiegl  mem       REG        8,6    27588   55216405 /usr/lib/libkrb5support.so.0.1
ssh-agent  7656     spiegl  mem       REG        8,6            15220746 /lib/i686/cmov/libdl-2.7.so (path inode=37373103)
ssh-agent  7656     spiegl  mem       REG        8,6            15220743 /lib/i686/cmov/libc-2.7.so (path inode=37373099)
ssh-agent  7656     spiegl  mem       REG        8,6            41811981 /lib/libcom_err.so.2.1 (path inode=41812032)
ssh-agent  7656     spiegl  mem       REG        8,6   143296   55216401 /usr/lib/libk5crypto.so.3.1
ssh-agent  7656     spiegl  mem       REG        8,6   582740   55216403 /usr/lib/libkrb5.so.3.3
ssh-agent  7656     spiegl  mem       REG        8,6   167960   55216316 /usr/lib/libgssapi_krb5.so.2.2
ssh-agent  7656     spiegl  mem       REG        8,6            15220745 /lib/i686/cmov/libcrypt-2.7.so (path inode=37373102)
ssh-agent  7656     spiegl  mem       REG        8,6            15220749 /lib/i686/cmov/libnsl-2.7.so (path inode=37373108)
ssh-agent  7656     spiegl  mem       REG        8,6            55215392 /usr/lib/libz.so.1.2.3.3 (path inode=55215466)
ssh-agent  7656     spiegl  mem       REG        8,6            15220762 /lib/i686/cmov/libutil-2.7.so (path inode=37373124)
ssh-agent  7656     spiegl  mem       REG        8,6            56181144 /usr/lib/i686/cmov/libcrypto.so.0.9.8 (path inode=56180982)
ssh-agent  7656     spiegl  mem       REG        8,6            15220758 /lib/i686/cmov/libresolv-2.7.so (path inode=37373120)
ssh-agent  7656     spiegl  mem       REG        8,6            41812019 /lib/ld-2.7.so (path inode=41811982)
ssh-agent  7656     spiegl    0u      CHR        1,3                  97 /dev/null
ssh-agent  7656     spiegl    1u      CHR        1,3                  97 /dev/null
ssh-agent  7656     spiegl    2u      CHR        1,3                  97 /dev/null
ssh-agent  7656     spiegl    3u     unix 0xf37196c0               15673 /tmp/ssh-KVjaJd7584/agent.7584
ssh-agent  7656     spiegl    5w     FIFO        0,5               14681 pipe
ssh-agent  7656     spiegl    7r     FIFO        0,5               14685 pipe


The parent xterm:
COMMAND     PID       USER   FD      TYPE     DEVICE     SIZE       NODE NAME
xterm     13138     spiegl  cwd       DIR        8,6    24576   20348937 /home/spiegl
xterm     13138     spiegl  rtd       DIR        8,6     4096          2 /
xterm     13138     spiegl  txt       REG        8,6   317648   15189861 /usr/bin/xterm
xterm     13138     spiegl  mem       REG        8,6    14396   55219117 /usr/lib/libXfixes.so.3.1.0
xterm     13138     spiegl  mem       REG        8,6    32764   55216382 /usr/lib/libXcursor.so.1.0.2
xterm     13138     spiegl  mem       REG        8,6  2283360   56968903 /usr/lib/locale/locale-archive
xterm     13138     spiegl  mem       REG        8,6   112354   37373119 /lib/i686/cmov/libpthread-2.7.so
xterm     13138     spiegl  mem       REG        8,6     9684   37373103 /lib/i686/cmov/libdl-2.7.so
xterm     13138     spiegl  mem       REG        8,6    16704   55220633 /usr/lib/libXdmcp.so.6.0.0
xterm     13138     spiegl  mem       REG        8,6     7220   55220621 /usr/lib/libXau.so.6.0.0
xterm     13138     spiegl  mem       REG        8,6    60776   37290193 /usr/lib/libXpm.so.4.11.0
xterm     13138     spiegl  mem       REG        8,6   126508   55216060 /usr/lib/libexpat.so.1.0.0
xterm     13138     spiegl  mem       REG        8,6    30544   55216369 /usr/lib/libXrender.so.1.3.0
xterm     13138     spiegl  mem       REG        8,6    81012   55215466 /usr/lib/libz.so.1.2.3.3
xterm     13138     spiegl  mem       REG        8,6   449240   55215678 /usr/lib/libfreetype.so.6.3.16
xterm     13138     spiegl  mem       REG        8,6  1356196   37373099 /lib/i686/cmov/libc-2.7.so
xterm     13138     spiegl  mem       REG        8,6   195564   41812031 /lib/libncurses.so.5.6
xterm     13138     spiegl  mem       REG        8,6   965952   55221003 /usr/lib/libX11.so.6.2.0
xterm     13138     spiegl  mem       REG        8,6    84880   28971387 /usr/lib/libICE.so.6.3.0
xterm     13138     spiegl  mem       REG        8,6    27944   55215460 /usr/lib/libSM.so.6.0.0
xterm     13138     spiegl  mem       REG        8,6   324484   55221345 /usr/lib/libXt.so.6.0.0
xterm     13138     spiegl  mem       REG        8,6    52848   55221313 /usr/lib/libXext.so.6.4.0
xterm     13138     spiegl  mem       REG        8,6    85528   55216599 /usr/lib/libXmu.so.6.2.0
xterm     13138     spiegl  mem       REG        8,6   363848   28971543 /usr/lib/libXaw7.so.7.0.0
xterm     13138     spiegl  mem       REG        8,6   168432   28968817 /usr/lib/libfontconfig.so.1.3.0
xterm     13138     spiegl  mem       REG        8,6    70088   55219304 /usr/lib/libXft.so.2.1.2
xterm     13138     spiegl  mem       REG        8,6     9480   55774283 /usr/lib/gconv/ISO8859-15.so
xterm     13138     spiegl  mem       REG        8,6    25700   55771214 /usr/lib/gconv/gconv-modules.cache
xterm     13138     spiegl  mem       REG        8,6   117344   41811982 /lib/ld-2.7.so
xterm     13138     spiegl    0r      CHR        1,3                  97 /dev/null
xterm     13138     spiegl    1w     FIFO        0,5               16122 pipe
xterm     13138     spiegl    2w     FIFO        0,5               16122 pipe
xterm     13138     spiegl    3u     unix 0xddbfdb80            37169512 socket
xterm     13138     spiegl    4r      CHR        1,3                  97 /dev/null
xterm     13138     spiegl    5u      CHR        5,2                 490 /dev/ptmx

The parent ssh:
COMMAND     PID       USER   FD      TYPE     DEVICE     SIZE       NODE NAME
ssh       13139     spiegl  cwd       DIR        8,6    24576   20348937 /home/spiegl
ssh       13139     spiegl  rtd       DIR        8,6     4096          2 /
ssh       13139     spiegl  txt       REG        8,6   288032   15188742 /usr/bin/ssh
ssh       13139     spiegl  mem       REG        8,6    38412   37373111 /lib/i686/cmov/libnss_files-2.7.so
ssh       13139     spiegl  mem       REG        8,6    34352   37373114 /lib/i686/cmov/libnss_nis-2.7.so
ssh       13139     spiegl  mem       REG        8,6    30436   37373109 /lib/i686/cmov/libnss_compat-2.7.so
ssh       13139     spiegl  mem       REG        8,6     5564   41812034 /lib/libkeyutils-1.2.so
ssh       13139     spiegl  mem       REG        8,6    27588   55216405 /usr/lib/libkrb5support.so.0.1
ssh       13139     spiegl  mem       REG        8,6     9684   37373103 /lib/i686/cmov/libdl-2.7.so
ssh       13139     spiegl  mem       REG        8,6  1356196   37373099 /lib/i686/cmov/libc-2.7.so
ssh       13139     spiegl  mem       REG        8,6     7364   41812032 /lib/libcom_err.so.2.1
ssh       13139     spiegl  mem       REG        8,6   143296   55216401 /usr/lib/libk5crypto.so.3.1
ssh       13139     spiegl  mem       REG        8,6   582740   55216403 /usr/lib/libkrb5.so.3.3
ssh       13139     spiegl  mem       REG        8,6   167960   55216316 /usr/lib/libgssapi_krb5.so.2.2
ssh       13139     spiegl  mem       REG        8,6    38300   37373102 /lib/i686/cmov/libcrypt-2.7.so
ssh       13139     spiegl  mem       REG        8,6    83708   37373108 /lib/i686/cmov/libnsl-2.7.so
ssh       13139     spiegl  mem       REG        8,6    81012   55215466 /usr/lib/libz.so.1.2.3.3
ssh       13139     spiegl  mem       REG        8,6     9696   37373124 /lib/i686/cmov/libutil-2.7.so
ssh       13139     spiegl  mem       REG        8,6  1338692   56180982 /usr/lib/i686/cmov/libcrypto.so.0.9.8
ssh       13139     spiegl  mem       REG        8,6    67408   37373120 /lib/i686/cmov/libresolv-2.7.so
ssh       13139     spiegl  mem       REG        8,6   117344   41811982 /lib/ld-2.7.so
ssh       13139     spiegl    0u      CHR     136,21                  23 /dev/pts/21
ssh       13139     spiegl    1u      CHR     136,21                  23 /dev/pts/21
ssh       13139     spiegl    2u      CHR     136,21                  23 /dev/pts/21
ssh       13139     spiegl    3u     IPv4   37169532                 TCP localhost:35553->localhost:ssh (ESTABLISHED)
ssh       13139     spiegl    4r      CHR        1,3                  97 /dev/null
ssh       13139     spiegl    8u     unix 0xe3d23180            37173322 socket

sshd      13140       root  cwd       DIR        8,6     4096          2 /
sshd      13140       root  rtd       DIR        8,6     4096          2 /
sshd      13140       root  txt       REG        8,6   359780   55148866 /usr/sbin/sshd
sshd      13140       root  DEL       REG        0,8            37169565 /dev/zero
sshd      13140       root  mem       REG        8,6    47300   12927146 /lib/security/pam_unix.so
sshd      13140       root  DEL       REG        0,8            37169556 /dev/zero
sshd      13140       root  mem       REG        8,6    38412   37373111 /lib/i686/cmov/libnss_files-2.7.so
sshd      13140       root  mem       REG        8,6    34352   37373114 /lib/i686/cmov/libnss_nis-2.7.so
sshd      13140       root  mem       REG        8,6    30436   37373109 /lib/i686/cmov/libnss_compat-2.7.so
sshd      13140       root  mem       REG        8,6     5564   41812034 /lib/libkeyutils-1.2.so
sshd      13140       root  mem       REG        8,6    27588   55216405 /usr/lib/libkrb5support.so.0.1
sshd      13140       root  mem       REG        8,6  1356196   37373099 /lib/i686/cmov/libc-2.7.so
sshd      13140       root  mem       REG        8,6     7364   41812032 /lib/libcom_err.so.2.1
sshd      13140       root  mem       REG        8,6   143296   55216401 /usr/lib/libk5crypto.so.3.1
sshd      13140       root  mem       REG        8,6   582740   55216403 /usr/lib/libkrb5.so.3.3
sshd      13140       root  mem       REG        8,6   167960   55216316 /usr/lib/libgssapi_krb5.so.2.2
sshd      13140       root  mem       REG        8,6    38300   37373102 /lib/i686/cmov/libcrypt-2.7.so
sshd      13140       root  mem       REG        8,6    83708   37373108 /lib/i686/cmov/libnsl-2.7.so
sshd      13140       root  mem       REG        8,6    81012   55215466 /usr/lib/libz.so.1.2.3.3
sshd      13140       root  mem       REG        8,6     9696   37373124 /lib/i686/cmov/libutil-2.7.so
sshd      13140       root  mem       REG        8,6  1338692   56180982 /usr/lib/i686/cmov/libcrypto.so.0.9.8
sshd      13140       root  mem       REG        8,6    67408   37373120 /lib/i686/cmov/libresolv-2.7.so
sshd      13140       root  mem       REG        8,6    91860   41811976 /lib/libselinux.so.1
sshd      13140       root  mem       REG        8,6     9684   37373103 /lib/i686/cmov/libdl-2.7.so
sshd      13140       root  mem       REG        8,6    37600   41811991 /lib/libpam.so.0.81.6
sshd      13140       root  mem       REG        8,6    31224   41812003 /lib/libwrap.so.0.7.6
sshd      13140       root  mem       REG        8,6    12208   12927039 /lib/security/pam_limits.so
sshd      13140       root  mem       REG        8,6     7808   12927043 /lib/security/pam_mail.so
sshd      13140       root  mem       REG        8,6     3708   12927045 /lib/security/pam_motd.so
sshd      13140       root  mem       REG        8,6     4260   12927133 /lib/security/pam_nologin.so
sshd      13140       root  mem       REG        8,6    10188   12927030 /lib/security/pam_env.so
sshd      13140       root  mem       REG        8,6   117344   41811982 /lib/ld-2.7.so
sshd      13140       root    0u      CHR        1,3                  97 /dev/null
sshd      13140       root    1u      CHR        1,3                  97 /dev/null
sshd      13140       root    2u      CHR        1,3                  97 /dev/null
sshd      13140       root    3r     IPv4   37169533                 TCP localhost:ssh->localhost:35553 (ESTABLISHED)
sshd      13140       root    4w     FIFO        0,5            37169566 pipe
sshd      13140       root    5u     FIFO        0,5            37169566 pipe
sshd      13140       root    7u     unix 0xf229b700            37169615 /tmp/ssh-RKePv13140/agent.13140
sshd      13140       root   11u     IPv4   37173321                 TCP localhost:6014->localhost:44057 (ESTABLISHED)

I also tried to find the strace lines where this socket is opened.
Should be this:
 munmap(0xb7f0e000, 4096)                = 0
 socket(PF_INET, SOCK_STREAM, IPPROTO_TCP) = 3
 connect(3, {sa_family=AF_INET, sin_port=htons(22), sin_addr=inet_addr("127.0.0.1")}, 16) = 0
 setsockopt(3, SOL_SOCKET, SO_KEEPALIVE, [1], 4) = 0

Lateron I found:
 read(3, "S", 1)                         = 1
 read(3, "S", 1)                         = 1
 read(3, "H", 1)                         = 1
 read(3, "-", 1)                         = 1
 read(3, "2", 1)                         = 1
 read(3, ".", 1)                         = 1
 read(3, "0", 1)                         = 1
 read(3, "-", 1)                         = 1
 read(3, "O", 1)                         = 1
 read(3, "p", 1)                         = 1
 read(3, "e", 1)                         = 1
 read(3, "n", 1)                         = 1
 read(3, "S", 1)                         = 1
 read(3, "S", 1)                         = 1
 read(3, "H", 1)                         = 1
 read(3, "_", 1)                         = 1
 read(3, "4", 1)                         = 1
 read(3, ".", 1)                         = 1
 read(3, "7", 1)                         = 1
 read(3, "p", 1)                         = 1
 read(3, "1", 1)                         = 1
 read(3, " ", 1)                         = 1
 read(3, "D", 1)                         = 1
 read(3, "e", 1)                         = 1
 read(3, "b", 1)                         = 1
 read(3, "i", 1)                         = 1
 read(3, "a", 1)                         = 1
 read(3, "n", 1)                         = 1
 read(3, "-", 1)                         = 1
 read(3, "3", 1)                         = 1
 read(3, "\n", 1)                        = 1
 write(3, "SSH-2.0-OpenSSH_4.7p1 Debian-3\n", 31) = 31
 fcntl64(3, F_GETFL)                     = 0x2 (flags O_RDWR)
 fcntl64(3, F_SETFL, O_RDWR|O_NONBLOCK)  = 0
 write(3, "\0\0\3\24\10\24\327l\7\245h\'~\0047\6\364\377\376\215\252"..., 792) = 792
 select(4, [3], NULL, NULL, NULL)        = 1 (in [3])
 read(3, "\0\0\3\f\n\24\322*9\1\225\30763\314\263\275\217y;\23\220"..., 8192) = 784
 write(3, "\0\0\0\24\6\"\0\0\4\0\0\0\4\0\0\0 \0\0\0\0\0\0\0", 24) = 24

That seems to be the ssh data connection itself, right?

Does this make sense to you?
 Andy.

-- 
 BREAKFAST.SYS halted... Cereal port not responding.



Messages sorted by: Reverse Date, Date, Thread, Author