dataTools/send72B.py

import json
import os
import requests

def process_json_files(input_file, output_folder, batch_size=20):
    # 确保输出文件夹存在，如果不存在则创建
    if not os.path.exists(output_folder):
        os.makedirs(output_folder)

    # 遍历输入文件夹中的JSON文件
    # for root, dirs, files in os.walk(input_folder):
    #     for i, file_name in enumerate(files, 1):
    #         if file_name.endswith('.json'):
    # input_file = os.path.join(root, file_name)
    # output_file = os.path.join(output_folder, file_name)
    file_name = os.path.basename(input_file)
    output_file = os.path.join(output_folder, file_name)
    process_single_json_file(input_file, output_file, batch_size)

def process_single_json_file(input_file, output_file, batch_size):
    # 计算文件的总行数
    with open(input_file, 'r', encoding='utf-8') as f_in:
        total_lines = sum(1 for line in f_in if line.strip())
        
    # 打开输入文件和输出文件
    with open(input_file, 'r', encoding='utf-8') as f_in, open(output_file, 'a', encoding='utf-8') as f_out:
        # 逐批处理输入文件中的JSON对象
        batch_data = []
        line_number = 0
        for line in f_in:
            line_number += 1
            # if line_number < 19319:
            #     continue
            # 加载JSON对象
            # data = json.loads(line)
            if not line.strip():
                continue
            batch_data.append(str(line))
            # 如果累积了足够的数据，就发送一次请求
            if len(batch_data) == batch_size:
                process_batch(batch_data, f_out)
                batch_data = []
                print(f"Processed {line_number} out of {total_lines} lines.")  # 打印进度
        # 处理最后一个不足一批的数据
        if batch_data:
            process_batch(batch_data, f_out)
            print(f"Processed {line_number} out of {total_lines} lines.")  # 打印进度
        
def process_batch(batch_data, f_out):
    
    # 构建请求数据
    request_data = {
        "max_tokens": 4096,
        "top_p": 1,
        "temperature": 0.6,
        "system_prompt": '''作为一位经验丰富的内容优化助手,根据输入的问答内容，你来丰富扩充答案内容，并分段或对重要内容加粗显示。
            ''',
        "querys":batch_data
    }
    
    # 发送请求
    response = requests.post("http://10.0.0.24:6000/v1/chat/completions", json=request_data)
    
    # 获取响应内容
    response_content = response.json()
    
    # 将响应内容写入输出文件
   
    for item in response_content['prediction']:
        # 获取当前问题
        question = batch_data.pop(0).strip()  # 移除并获取列表中的第一个问题
        json_obj = json.loads(question)
        # 构建问答格式的数据
        qa_pair = [json_obj[0], "答：" + item.strip()]
        # 将问答数据写入输出文件
        json.dump(qa_pair, f_out, ensure_ascii=False)
        f_out.write('\n')  # 写入换行符，以便下一组数据

def getQwen72():
    # 指定输入文件夹和输出文件夹
    input_folder = '/Users/yushanghui/hongshantianping/ai训练/data/shuiwuBook/jsonOut/2024.5.13/short.json'
    output_folder = '/Users/yushanghui/hongshantianping/ai训练/data/shuiwuBook/jsonOut/to72B-result/short5.13.json'

    # 处理JSON文件，设置每批次大小为20行
    process_json_files(input_folder, output_folder, batch_size=50)

def ceshi():
    rows = [{"ID":929652,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 20","TEST_TIME":"2024/03/02 20:00","COD_in":871.394653,"COD_in_name":630,"COD_off":15.017361,"COD_off_name":40,"COD_off_con":32,"TP_in":7.356771,"TP_in_name":7.1,"TP_off":0.065972,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":64.322914,"TN_in_name":65,"TN_off":8.090279,"TN_off_name":15,"TN_off_con":12,"andan_in":50.520832,"andan_in_name":58,"andan_off":0.729167,"andan_off_name":4,"andan_off_con":3.2,"SS_in":268.75,"SS_in_name":315,"SS_off":1.591435,"SS_off_name":10,"SS_off_con":8,"TDS_in":622.106506,"TDS_in_name":600,"T":"","pH":7.429398,"Q_in":2311,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[3796.875000,3303.819336]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.907813]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929592,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 19","TEST_TIME":"2024/03/02 19:00","COD_in":871.394653,"COD_in_name":630,"COD_off":14.612267,"COD_off_name":40,"COD_off_con":32,"TP_in":7.356771,"TP_in_name":7.1,"TP_off":0.07581,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":64.322914,"TN_in_name":65,"TN_off":9.0625,"TN_off_name":15,"TN_off_con":12,"andan_in":52.118057,"andan_in_name":58,"andan_off":1.342592,"andan_off_name":4,"andan_off_con":3.2,"SS_in":271.527771,"SS_in_name":315,"SS_off":1.50463,"SS_off_name":10,"SS_off_con":8,"TDS_in":694.444458,"TDS_in_name":600,"T":"","pH":7.405093,"Q_in":2394,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[3734.375000,3287.905029]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.916406]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929532,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 18","TEST_TIME":"2024/03/02 18:00","COD_in":871.394653,"COD_in_name":630,"COD_off":13.435184,"COD_off_name":40,"COD_off_con":32,"TP_in":7.581018,"TP_in_name":7.1,"TP_off":0.084201,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":68.923615,"TN_in_name":65,"TN_off":9.418403,"TN_off_name":15,"TN_off_con":12,"andan_in":56.875,"andan_in_name":58,"andan_off":0.619213,"andan_off_name":4,"andan_off_con":3.2,"SS_in":260.648132,"SS_in_name":315,"SS_off":2.517361,"SS_off_name":10,"SS_off_con":8,"TDS_in":494.791687,"TDS_in_name":600,"T":"","pH":7.441551,"Q_in":2350,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[3822.916504,3173.321777]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.925000]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929472,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 17","TEST_TIME":"2024/03/02 17:00","COD_in":871.394653,"COD_in_name":630,"COD_off":12.824074,"COD_off_name":40,"COD_off_con":32,"TP_in":7.581018,"TP_in_name":7.1,"TP_off":0.083044,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":68.923615,"TN_in_name":65,"TN_off":9.175347,"TN_off_name":15,"TN_off_con":12,"andan_in":54.652779,"andan_in_name":58,"andan_off":1.064815,"andan_off_name":4,"andan_off_con":3.2,"SS_in":316.898163,"SS_in_name":315,"SS_off":1.880787,"SS_off_name":10,"SS_off_con":8,"TDS_in":494.791687,"TDS_in_name":600,"T":"","pH":7.510417,"Q_in":2522,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[3739.583252,3170.138916]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.928125]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929412,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 16","TEST_TIME":"2024/03/02 16:00","COD_in":345.098389,"COD_in_name":630,"COD_off":11.525463,"COD_off_name":40,"COD_off_con":32,"TP_in":7.125289,"TP_in_name":7.1,"TP_off":0.069155,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":56.293404,"TN_in_name":65,"TN_off":9.53993,"TN_off_name":15,"TN_off_con":12,"andan_in":52.708332,"andan_in_name":58,"andan_off":1.006944,"andan_off_name":4,"andan_off_con":3.2,"SS_in":535.648193,"SS_in_name":315,"SS_off":2.40162,"SS_off_name":10,"SS_off_con":8,"TDS_in":454.28244,"TDS_in_name":600,"T":"","pH":7.502315,"Q_in":3242,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[3807.291748,3345.196777]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.903125]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929352,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 15","TEST_TIME":"2024/03/02 15:00","COD_in":345.381927,"COD_in_name":630,"COD_off":10.422453,"COD_off_name":40,"COD_off_con":32,"TP_in":7.125289,"TP_in_name":7.1,"TP_off":0.097512,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":56.293404,"TN_in_name":65,"TN_off":10.086805,"TN_off_name":15,"TN_off_con":12,"andan_in":48.333336,"andan_in_name":58,"andan_off":1.006944,"andan_off_name":4,"andan_off_con":3.2,"SS_in":274.074097,"SS_in_name":315,"SS_off":2.083333,"SS_off_name":10,"SS_off_con":8,"TDS_in":497.685181,"TDS_in_name":600,"T":"","pH":7.486111,"Q_in":3314,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[3942.708252,3287.905029]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.878125]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929292,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 14","TEST_TIME":"2024/03/02 14:00","COD_in":269.953705,"COD_in_name":630,"COD_off":10.724074,"COD_off_name":40,"COD_off_con":32,"TP_in":5.454282,"TP_in_name":7.1,"TP_off":0.097512,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":52.777779,"TN_in_name":65,"TN_off":10.086805,"TN_off_name":15,"TN_off_con":12,"andan_in":46.076389,"andan_in_name":58,"andan_off":1.006944,"andan_off_name":4,"andan_off_con":3.2,"SS_in":258.101868,"SS_in_name":315,"SS_off":2.199074,"SS_off_name":10,"SS_off_con":8,"TDS_in":457.175934,"TDS_in_name":600,"T":"","pH":7.490162,"Q_in":3408,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[4010.416748,3332.465332]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.880469]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929232,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 13","TEST_TIME":"2024/03/02 13:00","COD_in":269.953705,"COD_in_name":630,"COD_off":10.63426,"COD_off_name":40,"COD_off_con":32,"TP_in":5.454282,"TP_in_name":7.1,"TP_off":0.106481,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":52.777779,"TN_in_name":65,"TN_off":10.720487,"TN_off_name":15,"TN_off_con":12,"andan_in":42.881947,"andan_in_name":58,"andan_off":1.006944,"andan_off_name":4,"andan_off_con":3.2,"SS_in":250,"SS_in_name":315,"SS_off":2.40162,"SS_off_name":10,"SS_off_con":8,"TDS_in":526.620361,"TDS_in_name":600,"T":"","pH":7.449653,"Q_in":3214,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[4010.416748,3459.780029]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.883594]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929172,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 12","TEST_TIME":"2024/03/02 12:00","COD_in":247.268509,"COD_in_name":630,"COD_off":10.512267,"COD_off_name":40,"COD_off_con":32,"TP_in":5.613426,"TP_in_name":7.1,"TP_off":0.10706,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":59.288197,"TN_in_name":65,"TN_off":10.47743,"TN_off_name":15,"TN_off_con":12,"andan_in":40.034721,"andan_in_name":58,"andan_off":1.006944,"andan_off_name":4,"andan_off_con":3.2,"SS_in":244.675919,"SS_in_name":315,"SS_off":4.398148,"SS_off_name":10,"SS_off_con":8,"TDS_in":399.305573,"TDS_in_name":600,"T":"","pH":7.413195,"Q_in":3419,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[4135.416504,3536.168945]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.878125]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""},{"ID":929112,"TEST_DATE":"2024/03/02","TEST_HOUR":"2024/03/02 11","TEST_TIME":"2024/03/02 11:00","COD_in":246.98494,"COD_in_name":630,"COD_off":10.425,"COD_off_name":40,"COD_off_con":32,"TP_in":5.606192,"TP_in_name":7.1,"TP_off":0.115741,"TP_off_name":0.5,"TP_off_con":0.3,"TN_in":59.3316,"TN_in_name":65,"TN_off":11.215278,"TN_off_name":15,"TN_off_con":12,"andan_in":43.506943,"andan_in_name":58,"andan_off":1.006944,"andan_off_name":4,"andan_off_con":3.2,"SS_in":283.101868,"SS_in_name":315,"SS_off":5.584491,"SS_off_name":10,"SS_off_con":8,"TDS_in":358.796295,"TDS_in_name":600,"T":"","pH":7.445602,"Q_in":3314,"Q_in_name":4166.67,"continuity":"","check_rise":"","time_tips":"","cs_continuity":"","SVI":[187.852223,188.139059],"SVI_con":170,"SV":[90,92],"SV_con":80,"MLVSS":[1987,2064],"MLSS":"[4432.291504,3389.756836]","MLVSS_wentai":"","MLSS_wentai":"","DO_ana":[0.24,0.23],"DO_O":"[2.100000,2.893750]","DO_que":[0.22,0.2],"DO_que_wentai":"","DO_ana_wentai":"","DO_O_wentai":"","BOD_in":"","c":"","b":"","BTP_wending":"","JS_TN_wentai":"","CL_P04_YCZ":"","TP_LV":"","TP_LV_continuity":""}]
    rows = json.dumps(rows)
    feedback ={"2_2":"正常"}
    feedback = json.dumps(feedback)
   
   # 构建请求数据
    request_data = {
         "question": "什么是cod"
    }
    url = "http://10.0.0.28:8080/grpc/inferStreamRag"
    
    headers = {
        "Accept": "text/event-stream",
        "Content-Type": "application/json"
    }

    # 发送POST请求
    response = requests.post(url, headers=headers, json=request_data, stream=True)
    
    # 获取响应内容
    for line in response.iter_lines():
        if line:
            # 处理每一行事件数据
            print(line.decode('utf-8'))
if __name__ == "__main__":
    ceshi()