import json import os import requests def process_json_files(input_file, output_folder, batch_size=20): # 确保输出文件夹存在，如果不存在则创建 if not os.path.exists(output_folder): os.makedirs(output_folder) # 遍历输入文件夹中的JSON文件 # for root, dirs, files in os.walk(input_folder): # for i, file_name in enumerate(files, 1): # if file_name.endswith('.json'): # input_file = os.path.join(root, file_name) # output_file = os.path.join(output_folder, file_name) file_name = os.path.basename(input_file) output_file = os.path.join(output_folder, file_name) process_single_json_file(input_file, output_file, batch_size) def process_single_json_file(input_file, output_file, batch_size): # 计算文件的总行数 with open(input_file, 'r', encoding='utf-8') as f_in: total_lines = sum(1 for line in f_in if line.strip()) # 打开输入文件和输出文件 with open(input_file, 'r', encoding='utf-8') as f_in, open(output_file, 'a', encoding='utf-8') as f_out: # 逐批处理输入文件中的JSON对象 batch_data = [] line_number = 0 for line in f_in: line_number += 1 # if line_number < 19319: # continue # 加载JSON对象 # data = json.loads(line) if not line.strip(): continue batch_data.append(str(line)) # 如果累积了足够的数据，就发送一次请求 if len(batch_data) == batch_size: process_batch(batch_data, f_out) batch_data = [] print(f"Processed {line_number} out of {total_lines} lines.") # 打印进度 # 处理最后一个不足一批的数据 if batch_data: process_batch(batch_data, f_out) print(f"Processed {line_number} out of {total_lines} lines.") # 打印进度 def process_batch(batch_data, f_out): # 构建请求数据 request_data = { "max_tokens": 4096, "top_p": 1, "temperature": 0.6, "system_prompt": '''作为一位经验丰富的内容优化助手,根据输入的问答内容，你来丰富扩充答案内容，并分段或对重要内容加粗显示。 ''', "querys":batch_data } # 发送请求 response = requests.post("http://10.0.0.24:6000/v1/chat/completions", json=request_data) # 获取响应内容 response_content = response.json() # 将响应内容写入输出文件 for item in response_content['prediction']: # 获取当前问题 question = batch_data.pop(0).strip() # 移除并获取列表中的第一个问题 json_obj = json.loads(question) # 构建问答格式的数据 qa_pair = [json_obj[0], "答：" + item.strip()] # 将问答数据写入输出文件 json.dump(qa_pair, f_out, ensure_ascii=False) f_out.write('\n') # 写入换行符，以便下一组数据 def getQwen72(): # 指定输入文件夹和输出文件夹 input_folder = '/Users/yushanghui/hongshantianping/ai训练/data/shuiwuBook/jsonOut/2024.5.13/short.json' output_folder = '/Users/yushanghui/hongshantianping/ai训练/data/shuiwuBook/jsonOut/to72B-result/short5.13.json' # 处理JSON文件，设置每批次大小为20行 process_json_files(input_folder, output_folder, batch_size=50) def ceshi(): rows = [{"ID":929652,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 20","TEST_TIME":"2024/03/02 20:00","COD_in":871.394653,"COD_in_name":630,"COD_off":15.017361,"COD_off_name":40,"COD_off_con":32,"TP_in":7.356771,"TP_in_name":7.1,"TP_off":0.065972,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":64.322914,"TN_in_name":65,"TN_off":8.090279,"TN_off_name":15,"TN_off_con":12,"andan_in":50.520832,"andan_in_name":58,"andan_off":0.729167,"andan_off_name":4,"andan_off_con":3.2,"SS_in":268.75,"SS_in_name":315,"SS_off":1.591435,"SS_off_name":10,"SS_off_con":8,"TDS_in":622.106506,"TDS_in_name":600,"T":"","pH":7.429398,"Q_in":2311,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[3796.875000,3303.819336]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.907813]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929592,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 19","TEST_TIME":"2024/03/02 19:00","COD_in":871.394653,"COD_in_name":630,"COD_off":14.612267,"COD_off_name":40,"COD_off_con":32,"TP_in":7.356771,"TP_in_name":7.1,"TP_off":0.07581,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":64.322914,"TN_in_name":65,"TN_off":9.0625,"TN_off_name":15,"TN_off_con":12,"andan_in":52.118057,"andan_in_name":58,"andan_off":1.342592,"andan_off_name":4,"andan_off_con":3.2,"SS_in":271.527771,"SS_in_name":315,"SS_off":1.50463,"SS_off_name":10,"SS_off_con":8,"TDS_in":694.444458,"TDS_in_name":600,"T":"","pH":7.405093,"Q_in":2394,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[3734.375000,3287.905029]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.916406]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929532,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 18","TEST_TIME":"2024/03/02 18:00","COD_in":871.394653,"COD_in_name":630,"COD_off":13.435184,"COD_off_name":40,"COD_off_con":32,"TP_in":7.581018,"TP_in_name":7.1,"TP_off":0.084201,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":68.923615,"TN_in_name":65,"TN_off":9.418403,"TN_off_name":15,"TN_off_con":12,"andan_in":56.875,"andan_in_name":58,"andan_off":0.619213,"andan_off_name":4,"andan_off_con":3.2,"SS_in":260.648132,"SS_in_name":315,"SS_off":2.517361,"SS_off_name":10,"SS_off_con":8,"TDS_in":494.791687,"TDS_in_name":600,"T":"","pH":7.441551,"Q_in":2350,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[3822.916504,3173.321777]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.925000]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929472,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 17","TEST_TIME":"2024/03/02 17:00","COD_in":871.394653,"COD_in_name":630,"COD_off":12.824074,"COD_off_name":40,"COD_off_con":32,"TP_in":7.581018,"TP_in_name":7.1,"TP_off":0.083044,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":68.923615,"TN_in_name":65,"TN_off":9.175347,"TN_off_name":15,"TN_off_con":12,"andan_in":54.652779,"andan_in_name":58,"andan_off":1.064815,"andan_off_name":4,"andan_off_con":3.2,"SS_in":316.898163,"SS_in_name":315,"SS_off":1.880787,"SS_off_name":10,"SS_off_con":8,"TDS_in":494.791687,"TDS_in_name":600,"T":"","pH":7.510417,"Q_in":2522,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[3739.583252,3170.138916]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.928125]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929412,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 16","TEST_TIME":"2024/03/02 16:00","COD_in":345.098389,"COD_in_name":630,"COD_off":11.525463,"COD_off_name":40,"COD_off_con":32,"TP_in":7.125289,"TP_in_name":7.1,"TP_off":0.069155,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":56.293404,"TN_in_name":65,"TN_off":9.53993,"TN_off_name":15,"TN_off_con":12,"andan_in":52.708332,"andan_in_name":58,"andan_off":1.006944,"andan_off_name":4,"andan_off_con":3.2,"SS_in":535.648193,"SS_in_name":315,"SS_off":2.40162,"SS_off_name":10,"SS_off_con":8,"TDS_in":454.28244,"TDS_in_name":600,"T":"","pH":7.502315,"Q_in":3242,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[3807.291748,3345.196777]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.903125]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929352,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 15","TEST_TIME":"2024/03/02 15:00","COD_in":345.381927,"COD_in_name":630,"COD_off":10.422453,"COD_off_name":40,"COD_off_con":32,"TP_in":7.125289,"TP_in_name":7.1,"TP_off":0.097512,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":56.293404,"TN_in_name":65,"TN_off":10.086805,"TN_off_name":15,"TN_off_con":12,"andan_in":48.333336,"andan_in_name":58,"andan_off":1.006944,"andan_off_name":4,"andan_off_con":3.2,"SS_in":274.074097,"SS_in_name":315,"SS_off":2.083333,"SS_off_name":10,"SS_off_con":8,"TDS_in":497.685181,"TDS_in_name":600,"T":"","pH":7.486111,"Q_in":3314,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[3942.708252,3287.905029]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.878125]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929292,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 14","TEST_TIME":"2024/03/02 14:00","COD_in":269.953705,"COD_in_name":630,"COD_off":10.724074,"COD_off_name":40,"COD_off_con":32,"TP_in":5.454282,"TP_in_name":7.1,"TP_off":0.097512,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":52.777779,"TN_in_name":65,"TN_off":10.086805,"TN_off_name":15,"TN_off_con":12,"andan_in":46.076389,"andan_in_name":58,"andan_off":1.006944,"andan_off_name":4,"andan_off_con":3.2,"SS_in":258.101868,"SS_in_name":315,"SS_off":2.199074,"SS_off_name":10,"SS_off_con":8,"TDS_in":457.175934,"TDS_in_name":600,"T":"","pH":7.490162,"Q_in":3408,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[4010.416748,3332.465332]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.880469]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929232,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 13","TEST_TIME":"2024/03/02 13:00","COD_in":269.953705,"COD_in_name":630,"COD_off":10.63426,"COD_off_name":40,"COD_off_con":32,"TP_in":5.454282,"TP_in_name":7.1,"TP_off":0.106481,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":52.777779,"TN_in_name":65,"TN_off":10.720487,"TN_off_name":15,"TN_off_con":12,"andan_in":42.881947,"andan_in_name":58,"andan_off":1.006944,"andan_off_name":4,"andan_off_con":3.2,"SS_in":250,"SS_in_name":315,"SS_off":2.40162,"SS_off_name":10,"SS_off_con":8,"TDS_in":526.620361,"TDS_in_name":600,"T":"","pH":7.449653,"Q_in":3214,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[4010.416748,3459.780029]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.883594]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929172,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 12","TEST_TIME":"2024/03/02 12:00","COD_in":247.268509,"COD_in_name":630,"COD_off":10.512267,"COD_off_name":40,"COD_off_con":32,"TP_in":5.613426,"TP_in_name":7.1,"TP_off":0.10706,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":59.288197,"TN_in_name":65,"TN_off":10.47743,"TN_off_name":15,"TN_off_con":12,"andan_in":40.034721,"andan_in_name":58,"andan_off":1.006944,"andan_off_name":4,"andan_off_con":3.2,"SS_in":244.675919,"SS_in_name":315,"SS_off":4.398148,"SS_off_name":10,"SS_off_con":8,"TDS_in":399.305573,"TDS_in_name":600,"T":"","pH":7.413195,"Q_in":3419,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[4135.416504,3536.168945]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.878125]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929112,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 11","TEST_TIME":"2024/03/02 11:00","COD_in":246.98494,"COD_in_name":630,"COD_off":10.425,"COD_off_name":40,"COD_off_con":32,"TP_in":5.606192,"TP_in_name":7.1,"TP_off":0.115741,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":59.3316,"TN_in_name":65,"TN_off":11.215278,"TN_off_name":15,"TN_off_con":12,"andan_in":43.506943,"andan_in_name":58,"andan_off":1.006944,"andan_off_name":4,"andan_off_con":3.2,"SS_in":283.101868,"SS_in_name":315,"SS_off":5.584491,"SS_off_name":10,"SS_off_con":8,"TDS_in":358.796295,"TDS_in_name":600,"T":"","pH":7.445602,"Q_in":3314,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[4432.291504,3389.756836]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.893750]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""}] rows = json.dumps(rows) feedback ={"2_2":"正常"} feedback = json.dumps(feedback) # 构建请求数据 request_data = { "question": "什么是cod" } url = "http://10.0.0.28:8080/grpc/inferStreamRag" headers = { "Accept": "text/event-stream", "Content-Type": "application/json" } # 发送POST请求 response = requests.post(url, headers=headers, json=request_data, stream=True) # 获取响应内容 for line in response.iter_lines(): if line: # 处理每一行事件数据 print(line.decode('utf-8')) if __name__ == "__main__": ceshi()