check_db3.py适用于Python3.8#encoding:utf-8#author:qbit#date:2022-05-10#summary:遍历检查db3文件是否正确，统计记录数importosimportsysimporttimeimportpprintimportsqlite3importtracebackfrommultiprocessingimportPool,LockSubProcNum=1#子进程数TableName='my_table_name'Db3Dir=r'F:\tmp'defInit4ProcOneFile(lock):r"""子进程初始化函数"""globalgLockgLock=lockdefProcOneFile(db3file:str):r'''当文件没有损坏时，返回sqlite文件中的记录数'''pid=os.getpid()ifSubProcNum>2:#多进程版本带gLock:print(f'pid:{pid},ProcOneFile{db3file}...')else:#单进程版本print(f'pid:{pid},ProcOneFile{db3file}...')conn=sqlite3.connect(db3file)cur=conn.cursor()sql=f'selectcount(*)from{TableName};'exMsg=''try:cur.execute(sql)row=cur.fetchone()除了Exceptionasex:exMsg=traceback.format_exc()ifSubProcNum>2:#带gLock的多进程版本：print(f'Errorfile:{db3file}\n{exMsg}')else:print(f'Errorfile:{db3file}\n{exMsg}')cur.关闭()conn.close()ifexMsg:return'no',0,db3fileelse:return'ok',row[0],db3fileif__name__=='__main__':db3List=[]forroot,dirs,filesinos.walk(Db3Dir):forfileinfiles:pathfile=os.path.join(root,file)ifnotfile.endswith('.db3'):continuedb3List.append(pathfile)print(f'db3Listsize:{len(db3List)}')ifnotdb3List:sys.exit(0)startTime=time.time()lock=Lock()print(f'Numberofsubprocesses:{SubProcNum}')okDb3List=[]#文件是未损坏，且数据项数大于0zeroDb3List=[]#文件可读，数据项数为0errDb3List=[]#如果SubProcNum>1，则文件已损坏：#多进程版本#多进程还没开，不用lockprint(f'{db3List=}')withPool(SubProcNum,initializer=Init4ProcOneFile,initargs=(lock,))asp:results=p.imap_unordered(func=ProcOneFile,iterable=db3List)recordNum=0#完整文件总数fileCnt=0#已处理文件数对于结果中的状态、num、db3file：ifstatus=='ok':ifnum>0:recordNum+=numokDb3List.append((num,db3file))else:zeroDb3List.append((num,db3file))else:errDb3List.append(db3file)fileCnt+=1withlock:print(f'要处理的文件总数：{len(db3List)},'f'处理的文件数：{fileCnt},'f'okDb3Listsize:{len(okDb3List)},errDb3Listsize:{len(errDb3List)}',f'elapsedtime:{(time.time()-startTime):.2f}s')print(f'required总数处理的文件：{len(db3List)}，'f'已处理文件数：{fileCnt}，'f'okDb3List大小：{len(okDb3List)}，errDb3List大小：{len(errDb3List)}'，f'已用时间：{(time.time()-startTime):.2f}s')print(f"okDb3List:\n{pprint.pformat(okDb3List)}")print(f"zeroDb3List:\n{pprint.pformat(zeroDb3List)}")打印(f"errDb3List:\n{pprint.pformat(errDb3List)}")打印(f"db3List大小：{len(db3List)}，okDb3List大小：{len(okDb3List)}，zeroDb3List大小：{len(zeroDb3List)},errDb3Listsize:{len(errDb3List)}")print(f"okDb3recordnumber:{recordNum}")else:#单进程版本print(f'{db3List=}')recordNum=0#完整的总数numberofdatainthefilefileCnt=0#db3List中db3file处理的文件数：status,num,db3file=ProcOneFile(db3file)ifstatus=='ok':ifnum>0：recordNum+=numokDb3List.append((num,db3file))else:zeroDb3List.append((num,db3file))else:errDb3List.append(db3file)fileCnt+=1withlock:print(f'totalNumberof文件：{len(db3List)}，'f'已处理文件数：{fileCnt}，'f'okDb3List大小：{len(okDb3List)}，errDb3List大小：{len(errDb3List)}'，f'经过时间：{(time.time()-startTime):.2f}s')print(f"okDb3List:\n{pprint.pformat(okDb3List)}")print(f"zeroDb3List:\n{pprint.pformat(zeroDb3List)}")print(f"errDb3List:\n{pprint.pformat(errDb3List)}")print(f"db3List大小：{len(db3List)}，okDb3List大小：{len(okDb3List)}，zeroDb3List大小：{len(zeroDb3List)},errDb3List大小:{len(errDb3List)}")print(f"okDb3记录数:{recordNum}")print(f"Timetotal:{(time.time()-startTime):.2f}s")本文来自qbitsnap
