The mpi API on metaserver at Caltech-Dev

Legend:

Normal status or debugging message

Notable condition which may be a non-fatal error

Error condition not fatal to job

Error condition which fatal to job

Notable condition which is not an error

Currently undefined

Condition requires email notification of the responsible administrator of this API

Condition requires phone notification of the responsible administrator of this API

Link: API Status Page for Caltech-Dev

941587522 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587522 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587522 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587522 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587522 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587522 CXX_THREAD fork_parent@MountPointStatus.cc end
941587522 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587522 CXX_THREAD fork_parent@MountPointStatus.cc end
941587523 STARTUP execOverload overloaded exec
941587523 STARTUP closeListenSock no cid registered for service 'data'
941587523 STARTUP mpi::init unused data port 10021 closed
941587523 STARTUP mpi::init port 10021 (jobstate) opened on metaserver as sock3
941587523 STARTUP bgLoop Looping process watchlogs started
941587523 STARTUP openListenSock port 10019 (operator) opened on metaserver as sock4
941587523 STARTUP openListenSock port 10020 (emergency) opened on metaserver as sock5
941587523 STARTUP setResourceLimit vmemoryuse=unlimited; datasize=unlimited; core=unlimited; maxproc=65536; descriptors=1024; memorylocked=32768; filesize=unlimited; cputime=unlimited
941587523 STARTUP leakLogger inital size of mpi API: 14348 kB
941587523 STARTUP bgLoop Looping process etchosts started
941587523 STARTUP mpi Trying API
941587523 STARTUP mpi API yes
941587523 STARTUP mpi Trying LDAS_SYSTEM
941587523 STARTUP mpi LDAS_SYSTEM yes
941587523 IDLE exec exec /usr/sbin/lsof -b -p 7623
941587523 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587523 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587523 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587523 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587523 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587523 CXX_THREAD fork_parent@MountPointStatus.cc end
941587523 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587523 CXX_THREAD fork_parent@MountPointStatus.cc end
941587523 IDLE bgLoop Looping process statpagefile started
941587523 IDLE bgLoop Looping process killedjobreaper started
941587527 IDLE setFTPandHTTPinfo (::FTPURL 'ftp://131.215.115.248') (::FTPDIR '') (::HTTPURL 'http://131.215.115.248/ldas_outgoing/jobs') (::HTTPDIR '/ldas_outgoing/jobs') (::GRIDFTPURL 'gridftp:/export/grid/ldas') (::GRIDFTPDIR '/export/grid/ldas') (::LDAS_GATEWAY 'ldas-dev 131.215.115.248') (::LDAS_SYSTEM 'ldas-dev') (::RUNCODE 'LDAS-DEV')
941587527 IDLE setFTPandHTTPinfo (::FTPURL 'ftp://131.215.115.248') (::FTPDIR '') (::HTTPURL 'http://131.215.115.248/ldas_outgoing/jobs') (::HTTPDIR '/ldas_outgoing/jobs') (::GRIDFTPURL 'gridftp:/export/grid/ldas') (::GRIDFTPDIR '/export/grid/ldas') (::LDAS_GATEWAY 'ldas-dev 131.215.115.248') (::LDAS_SYSTEM 'ldas-dev') (::RUNCODE 'LDAS-DEV')
941587528 STARTUP mpi::killAllMpirun cleaning up for user ldas
941587528 IDLE ::exec exec /usr/bin/ssh -n -l ldas beowulf /bin/rm -rf /tmp/lam-ldas@*
941587528 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587528 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587528 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587528 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587528 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587528 CXX_THREAD fork_parent@MountPointStatus.cc end
941587528 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587528 CXX_THREAD fork_parent@MountPointStatus.cc end
941587531 IDLE ::exec exec /usr/bin/ssh -n beowulf sudo pkill -9 -u search01,search02,search03,search04,search05,search06,search07,search08,search09,search10,search11,search12,search13,search14,search15,search16 wrapperAPI
941587531 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587531 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587531 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587531 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587531 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587531 CXX_THREAD fork_parent@MountPointStatus.cc end
941587531 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587531 CXX_THREAD fork_parent@MountPointStatus.cc end
941587534 IDLE ::exec exec /usr/bin/ssh -n beowulf sudo pkill -9 -u search01,search02,search03,search04,search05,search06,search07,search08,search09,search10,search11,search12,search13,search14,search15,search16 lamd
941587534 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587534 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587534 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587534 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587534 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587534 CXX_THREAD fork_parent@MountPointStatus.cc end
941587534 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587534 CXX_THREAD fork_parent@MountPointStatus.cc end
941587537 STARTUP mpi::killAllMpirun ran kill 10 times in 9.042 seconds
941587537 STARTUP mpi::prestartLamds running lamboot for user search01
941587537 IDLE ::exec exec scp -B conf.lam search01@metaserver:
941587537 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587537 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587537 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587537 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587537 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587537 CXX_THREAD fork_parent@MountPointStatus.cc end
941587537 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587537 CXX_THREAD fork_parent@MountPointStatus.cc end
941587537 IDLE ::exec exec scp -B conf.lam search01@metaserver:
941587537 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587537 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587537 CXX_THREAD fork_prepare@MountPointStatus.cc begin
941587537 CXX_THREAD fork_prepare@MountPointStatus.cc end
941587537 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587537 CXX_THREAD fork_parent@MountPointStatus.cc end
941587537 CXX_THREAD fork_parent@MountPointStatus.cc begin
941587537 CXX_THREAD fork_parent@MountPointStatus.cc end
941587537 STARTUP mpi::prestartLamds Subject: LDAS-DEV mpiAPI lamboot error!; Body: lam::boot: lam::pushSchema: search01@metaserver: Permission denied (publickey,gssapi-with-mic,password). lost connection
941587537 STARTUP mpi::killAllMpirun {mpi::cleanupTmp: ldas@beowulf: ssh: connect to host beowulf port 22: No route to host} {ldas@beowulf:wrapperAPI: ssh: connect to host beowulf port 22: No route to host} {ldas@beowulf:lamd: ssh: connect to host beowulf port 22: No route to host}
941587543 IDLE mpi::updateCmonNodelist updated ::beowulfNodes in cntlmonAPI to 'beowulf beowulf beowulf beowulf beowulf beowulf beowulf beowulf beowulf'
941587548 IDLE mpi::updateCmonNodelist updated ::beowulfNodes in cntlmonAPI to 'beowulf beowulf beowulf beowulf beowulf beowulf beowulf beowulf beowulf'
941764771 IDLE mpi::reply Manager already hung up on socket: 'sock7'.
941844781 IDLE mpi::reply Manager already hung up on socket: 'sock7'.
942094831 IDLE mpi::reply Manager already hung up on socket: 'sock7'.
942621702 SHUTDOWN closeListenSock port 10019 (sock4) (operator) closed on metaserver
942621702 SHUTDOWN mpi::sHuTdOwN Subject: LDAS Caltech-Dev mpi shutdown at 942621701 ( 11/18/2009 03:21:26 PM PST ); Body: mpi shutting down NOW , nocore
942621702 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search01'
942621702 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search01 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
942621702 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621702 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621702 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621702 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621702 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621702 CXX_THREAD fork_parent@MountPointStatus.cc end
942621702 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621702 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
942621703 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search02'
942621703 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search02 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
942621703 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search03'
942621703 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search03 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
942621703 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search04'
942621703 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search04 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
942621703 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search05'
942621703 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search05 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
942621703 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search06'
942621703 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search06 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
942621703 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search07'
942621703 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search07 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
942621703 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search08'
942621703 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search08 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
942621703 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search09'
942621703 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search09 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
942621703 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search10'
942621703 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search10 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621703 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621703 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621703 CXX_THREAD fork_parent@MountPointStatus.cc end
942621704 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
942621704 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search11'
942621704 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search11 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
942621704 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621704 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621704 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621704 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621704 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621704 CXX_THREAD fork_parent@MountPointStatus.cc end
942621704 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621704 CXX_THREAD fork_parent@MountPointStatus.cc end
942621704 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
942621704 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search12'
942621704 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search12 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
942621704 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621704 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621704 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621704 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621704 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621704 CXX_THREAD fork_parent@MountPointStatus.cc end
942621704 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621704 CXX_THREAD fork_parent@MountPointStatus.cc end
942621704 SHUTDOWN lam::halt Permission denied, please try again. Permission denied, please try again. Permission denied (publickey,gssapi-with-mic,password).
942621704 SHUTDOWN ::mpi::atExit calling lam::halt for user 'search13'
942621704 SHUTDOWN ::exec exec /usr/bin/ssh -n -l search13 metaserver /usr/bin/env LAMHOME=/usr/lib/lam LAMRSH=/usr/bin/ssh /usr/lib/lam/bin/lamhalt
942621704 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621704 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621704 CXX_THREAD fork_prepare@MountPointStatus.cc begin
942621704 CXX_THREAD fork_prepare@MountPointStatus.cc end
942621704 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621704 CXX_THREAD fork_parent@MountPointStatus.cc end
942621704 CXX_THREAD fork_parent@MountPointStatus.cc begin
942621704 CXX_THREAD fork_parent@MountPointStatus.cc end