本文转载自微信公众号《运维开发故事》，作者的运维开发故事。转载本文请联系运维开发故事公众号。1.前言排错过程，源码部分由我开发同事查看记录；征得他们的同意后，我会在这里发布。2.问题一天收到客户反馈，出现大量warning事件：Readinessprobefailed:OCIruntimeexecfailed:execfailed:EOF:unknown出现在pod事件中。但不影响客户使用服务。3、环境特别说明：客户坚持在负责运行业务的k8s节点上开启cpu-manager组件版本k8s1.14.x。四、检查1、收到客户反馈后，检查pod所在节点的kubelet日志，如下：I050703:43:28.31063057003prober.go:112]Readinessprobefor"adsfadofadfabdfhaodsfa(d1aab5f0-ae8f-11eb-a151-080027049c65):c0"failed(failure):OCIruntimeexecfailed:execfailed:EOF:unknownI050707:08:49.83409357003prober.go:112]"adsfadofadfabdfhaodsfa(a89a158e-ae8f-11eb-a151-080027049c65):failocurec65):failocurec65:failocurec65execfailed：unexpectedEOF：unknownI050710：06：58.30788157003prober.go：112]Readinessprobefor“adsfadofadfabdfhaodsfa（d1aab5f0-ae8f-11eb-a151-080027049c65）：c0”失败（失败）：OCIruntimeexecfailuned：错误类型：EOF失败：EOF失败：对应代码如下：图2.再次查看docker日志，如下：time="2021-05-06T16:51:40.009989451+08:00"level=errormsg="streamcopyerror:readingfromaclosedfifo"time="2021-05-06T16:51:40.010054596+08:00"level=errormsg="streamcopyerror:readingfromaclosedfifo"time="2021-05-06T16:51:40.170676532+08:00"level=errormsg="Errorrunningexec8e34e8b910694abe95a467b2936b37635fdabd2f7b7c464dfef952fa5732aa4eincontainer:OCIruntimeexecfailed:execfailed:EOF:unknown"从Docker日志中显示为流复制错误，但实际上底层runc返回了EOF，导致错误3，因为日志显示探测类型为Failure，所以e.CombinedOutPut()err!=nil，且ExitStatus不为0，data值为OCIruntimeexecfailed:execfailed:unexpectedEOF:unknown，最终会调用RunInContainer方法，ExecSync是通过GRPC.Dockershim调用dockershim的ExecSync最后调用ExecInContainer方法，该方法返回exitcode不为0的错误。func(*NativeExecHandler)ExecInContainer(clientlibdocker.Interface,container*dockertypes.ContainerJSON,cmd[]string,stdinio.Reader,stdout,stderrio.WriteCloser,ttybool,resize<-chanremotecommand.TerminalSize,timeouttime.Duration)错误{execObj,err:=client.CreateExec(container.ID,createOpts)startOpts:=dockertypes.ExecStartCheck{Detach:false,Tty:tty}streamOpts:=libdocker.StreamOptions{InputStream:stdin,OutputStream:stdout,ErrorStream:stderr,RawTerminal:tty,ExecStarted:execStarted,}err=client.StartExec(execObj.ID,startOpts,streamOpts)iferr!=nil{returnerr}ticker:=time.NewTicker(2*time.Second)deferticker.Stop()count:=0for{检查,err2:=client.InspectExec(execObj.ID)iferr2!=nil{returnerr2}if!inspect.Running{ifinspect.ExitCode!=0{err=&dockerExitError{inspect}}break}count++ifcount==5{klog.Errorf("Execsession%sincontainer%sterminatedbutprocessstillrunning!",execObj.ID,container.ID)break}<-ticker.C}returnerr}ExecInContainer做了下面几个事情：调用CreatExec创建ExecID，调用StartExec执行exec，通过holdHijackedConnection重定向输入输出，将inputStream写入connection，将响应流重定向到stdout、stderr。调用InspectExec获取这个exec的运行状态和exitcode，那么log中打印的错误信息就是response流传过来的字符流。换句话说，dockerd的响应包含错误值。这时候去docker代码找原因，ExecStart会调用dockerd的如下代码：根据上面的dockerlog，err的错误信息是：OCIruntimeexecfailed:execfailed:EOF:unknown。也就是说，ContainerExecStart返回了一个错误。ContainerExecStart会调用到containerd.Exec，也就是dockerd和containerd之间进行通信//docker/libcontainerd/client_daemon.go//Execcreatesexecprocess.////ThecontainerdclientcallsExectoregistertheexecconfigintheshimside.//WhentheclientcallsStart,theshimwillcreatestdinfifoifneeds.But//forthecontainermainprocess,thestdinfifowillbecreatedinCreatenot//Startcall.stdinCloseSyncchannelshouldbeclosedafterStartexec//process.func(c*client)Exec(ctxcontext.Context,containerID,processIDstring,spec*specs.Process,withStdinbool,attachStdioStdioCallback)(int,error){ctr:=c.getContainer(containerID)ifctr==nil{return-1,errors.WithStack(newNotFoundError("nosuchcontainer"))}t:=ctr.getTask()ift==nil{return-1,errors.WithStack(newInvalidParameterError("containerisnotrunning"))}ifp:=ctr.getProcess(processID);p!=nil{return-1,errors.WithStack(newConflictError("idalreadyinuse"))}var(pcontainerd.Processriocio.IOerrerrorstdinCloseSync=make(chanstruct{}))fifos:=newFIFOSet(ctr.bundleDir,processID,withStdin,spec.Terminal)deferfunc(){iferr!=nil{ifrio!=nil{rio.Cancel()rio.Close()}}}()p,err=t。Exec(ctx,processID,spec,func(idstring)(cio.IO,error){rio,err=c.createIO(fifos,containerID,processID,stdinCloseSync,attachStdio)returnrio,err})iferr!=nil{close(stdinCloseSync)return-1,wrapError(err)}ctr.addProcess(processID,p)//Signalc.createIOthatitcancallCloseIO////thestdinofexecprocesswillbecreatedafterp.Startincontainerddeferclose(stdinCloseSync)iferr=p.Start(ctx);err!=nil{//使用新上下文进行清理，因为旧的可能会被用户取消，但要留出超时时间以确保//我们不会永远等待容器无响应或解决//旧容器中的fifocancelling问题d-shimctx，取消：=context.WithTimeout（context.Background（），45*time.Second）defercancel（）p.Delete（ctx）ctr.delete）return-ID（进程ID）wrapError(err)}returnint(p.Pid()),nil}这里new了一个FIFOSet，而readingfromaclosedfifo只出现在fifo被close掉时，仍然在读取的情况也就是说，f.Close()发生在f.Read()之前。在外层可以看到deferfunc(){iferr!=nil{ifrio!=nil{rio.Cancel()rio.Close()//这里关闭会导致fifoclose}}}()p,err=t。执行（ctx，processID，spec，func（idstring）（cio.IO，error）{rio，err=c.createIO（fifos，containerID，processID，stdinCloseSync，attachStdio）returnrio，err}）iferr！=nil{close（stdinCloseSync)return-1,wrapError(err)}ctr.addProcess(processID,p)//Signalc.createIOthatitcancallCloseIO////thestdinoexecprocesswillbecreatedafterp.Startincontainerddeferclose(stdinCloseSync)//p.Start出错，会导致内部fifo关闭，导致从关闭的fifo中读取的问题(),p.Delete(ctx)ctr.deleteProcess(processID)return-1,wrapError(err)}p.Start调用如下代码，co通过GRPC与containerd通信。//github.com/containerd/containerd/task.gofunc(t*task)Start(ctxcontext.Context)error{r,err:=t.client.TaskService().Start(ctx,&tasks.StartRequest{ContainerID:t...",e.id)))ifestdio.Terminal{ifsocket,err=runc.NewTempConsoleSocket();err!=nil{returnerrors.Wrap(err,"failedtocreateruncconsoleSocket")}defersocket.Close()}elseife.stdio.IsNull(){ife.io,err=runc.NewNullIO();err!=nil{returnsrors.Wrap(err,"creatingnewNULLIO")}}else{ife.io,err=runc.NewPipeIO(e.parent.IoUID,e.parent.IoGID,withConditionalIO(e.stdio));.ConsoleSocket=套接字}。//errreturnedexecfailed:EOF:unknown//这里的runtime是runc执行命令的二进制文件iferr:=e.parent.runtime.Exec(ctx,e.parent.id,e.spec,opts);呃！=nil{close(e.waitBlock)return.parent.runtimeError(err,"OCIruntimeexecfailed")}Exec的代码如下：//Execexecutresandadditionalprocessinsidethecontainerbasedonafull//OCIProcessspecificationfunc(r*Runc)Exec(contextcontext.Context,idstring,specspecs.Process,opts*ExecOpts)error{f,err:=ioutil.TempFile(os.Getenv("XDG_RUNTIME_DIR"),"runc-process")iferr!=nil{returnerr}deferos.Remove(f.Name())err=json.NewEncoder(f).Encode(spec)f.Close()iferr!=nil{returnerr}args:=[]string{"exec","--process",f.Name()}ifopts!=nil{oargs,err:=opts.args()iferr!=nil{returnerr}args=append(args,oargs...)}cmd:=r.command(context,append(args,id)...)ifopts！=nil&&opts.IO!=nil{opts.Set(cmd)}ifcmd.Stdout==nil&&cmd.Stderr==nil{data,err:=cmdOutput(cmd,true)iferr!=nil{returnfmt.Errorf("%s:%s",错误,数据)}returnnil}ec,err:=Monitor.Start(cmd)iferr!=nil{returnerr}ifopts!=nil&&opts.IO!=nil{ifc,ok:=opts.IO.(StartCloser);ok{iferr:=c.CloseAfterStart();err!=nil{returnerr}}}status,err:=Monitor.Wait(cmd,ec)iferr==nil&&status!=0{err=fmt.Errorf("%sdidnotterminatesucessfully",cmd.Args[0])}returnerr}因此runc运行后输出execfailed:EOF:unknown。该错误循环执行runc命令，可少量复现。经过排查，发现runcexec运行时会读取容器的state.json，使用jsondecode时会出现异常。这时想到kubeletcpu-manager启动后，容器会更新，也就是state.json文件会更新。导致runc读取部分cpu-manager更新。结果，json解码失败。此时查看runcEOF和kubeletcpu-managerupdatecontainer的时间（默认10s更新一次），发现时间点完全一致，验证了猜想。检查是否有针对runc的修复，并找到了这个pr：https://github.com/opencontainers/runc/pull/2467。修复思路是把saveState变成一个原子操作，这样在读取state.json的时候，有些写入的内容不会被读取，导致意外EOF（或者EOF）的问题//原来的func(c*linuxContainer)saveState(s*State)error{f,err:=os.Create(filepath.Join(c.root,stateFilename))iferr!=nil{returnerr}deferf.Close()returnutils.WriteJSON(f,s)}//修复func(c*linuxContainer)saveState(s*State)(retErrerror){tmpFile,err:=ioutil.TempFile(c.root,"state-")iferr!=nil{returnerr}deferfunc(){ifretErr!=nil{tmpFile.Close()os.Remove(tmpFile.Name())}}()err=utils.WriteJSON(tmpFile,s)iferr!=nil{returnerr}err=tmpFile.Close()iferr!=nil{returnerr}stateFilePath:=filepath.Join(c.root,stateFilename)returnos.Rename(tmpFile.Name(),stateFilePath)}五、解决关闭cpu-manager升级runc的问题
