voice-gateway/tests/pjsua/runall.py

import os
import sys
import time
import re
import shutil
import platform

PYTHON = os.path.basename(sys.executable)

# Usage:
#  runall.py [test-to-resume]


# Initialize test list
tests = []

# Excluded tests (because they fail?)
excluded_tests = [
    "svn",
    "pyc",
    "scripts-call/150_srtp_2_1",                     # SRTP optional 'cannot' call SRTP mandatory
    "scripts-call/150_srtp_2_3.py",                  # disabled because #1267 wontfix
    "scripts-call/301_ice_public_a.py",              # Unreliable, proxy returns 408 sometimes
    "scripts-call/301_ice_public_b.py",              # Doesn't work because OpenSER modifies SDP
    "scripts-pres/200_publish.py",                   # Ok from cmdline, error from runall.py
    "scripts-media-playrec/100_resample_lf_8_11.py", # related to clock-rate 11 kHz problem
    "scripts-media-playrec/100_resample_lf_8_22.py", # related to clock-rate 22 kHz problem
    "scripts-media-playrec/100_resample_lf_11",      # related to clock-rate 11 kHz problem
    "pesq",                                          # temporarily disabling all pesq related test due to unreliability
    # TODO check all tests below for false negatives
    "call_305_ice_comp_1_2",
    "scripts-sendto/155_err_sdp_bad_syntax",
    "transfer-attended",
    "uac-inv-and-ack-without-sdp",
    "uac-subscribe",
    "uac-ticket-1148",
    "uas-422-then-200-bad-se",
    "uas-answer-180-multiple-fmts-support-update",
    "uas-inv-answered-with-srtp",
    "uas-mwi-0",
    "uas-mwi",
    "uas-register-ip-change-port-only",
    "uas-register-ip-change",
    "uas-timer-update"
]

# Exclude scripts-sipp/uac-reinvite-bad-via-branch on MacOS due to unreliable result
if platform.system()=='Darwin':
    excluded_tests.append("scripts-sipp/uac-reinvite-bad-via-branch")


# Add basic tests
for f in os.listdir("scripts-run"):
    tests.append("mod_run.py scripts-run/" + f)

# Add basic call tests
for f in os.listdir("scripts-call"):
    tests.append("mod_call.py scripts-call/" + f)

# Add presence tests
for f in os.listdir("scripts-pres"):
    tests.append("mod_pres.py scripts-pres/" + f)

# Add mod_sendto tests
for f in os.listdir("scripts-sendto"):
    tests.append("mod_sendto.py scripts-sendto/" + f)

# Add mod_media_playrec tests
for f in os.listdir("scripts-media-playrec"):
    tests.append("mod_media_playrec.py scripts-media-playrec/" + f)

# Add mod_pesq tests
for f in os.listdir("scripts-pesq"):
    tests.append("mod_pesq.py scripts-pesq/" + f)

# Add recvfrom tests
for f in os.listdir("scripts-recvfrom"):
    tests.append("mod_recvfrom.py scripts-recvfrom/" + f)

# Add sipp tests
for f in os.listdir("scripts-sipp"):
    if f.endswith(".xml"):
        tests.append("mod_sipp.py scripts-sipp/" + f)


resume_script=""
retry_num=0
shell_cmd=""
with_log=True

# Parse arguments
sys.argv.pop(0)
while len(sys.argv):
    if sys.argv[0]=='/h' or sys.argv[0]=='-h' or sys.argv[0]=='--help' or sys.argv[0]=='/help':
        sys.argv.pop(0)
        print("Usage:")
        print("  runall.py [OPTIONS] [run.py-OPTIONS]")
        print("OPTIONS:")
        print("  --list")
        print("    List the tests")
        print("  --list-xml")
        print("    List the tests as XML format suitable for ccdash")
        print("  --resume,-r RESUME")
        print("    RESUME is string/substring to specify where to resume tests.")
        print("    If this argument is omited, tests will start from the beginning.")
        print("  --disable,-d TEST_NAME")
        print("    Disable a specific test that contains the specified TEST_NAME.")
        print("  --retry,-t RETRY_NUM")
        print("    Retry a specific test RETRY_NUM times before marking it as failed.")
        print("    Default is 0 (no retry).")
        print("  --shell,-s SHELL")
        print("    Run the tests with the specified SHELL cmd. This can also be")
        print("    used to run the test with ccdash. Example:")
        print("    --shell '/bin/sh -c'")
        print("  --no-log")
        print("    Do not generate log files. By default log files will be generated")
        print("    and put in 'logs' dir.")
        print("")
        print("  run.py-OPTIONS are applicable here")
        sys.exit(0)
    elif sys.argv[0] == '-r' or sys.argv[0] == '--resume':
        if len(sys.argv) > 1:
            resume_script=sys.argv[1]
            sys.argv.pop(0)
            sys.argv.pop(0)
        else:
            sys.argv.pop(0)
            sys.stderr.write("Error: argument value required")
            sys.exit(1)
    elif sys.argv[0] == '-t' or sys.argv[0] == '--retry':
        if len(sys.argv) > 1:
            retry_num=int(sys.argv[1])
            sys.argv.pop(0)
            sys.argv.pop(0)
        else:
            sys.argv.pop(0)
            sys.stderr.write("Error: argument value required")
            sys.exit(1)
    elif sys.argv[0] == '--list':
        sys.argv.pop(0)
        for t in tests:
              print(t)
        sys.exit(0)
    elif sys.argv[0] == '--list-xml':
        sys.argv.pop(0)
        for t in tests:
            (mod,param) = t.split(None,2)
            tname = mod[4:mod.find(".py")] + "_" + \
            param[param.find("/")+1:param.rfind(".")]
            c = ""
            if len(sys.argv):
                c = " ".join(sys.argv) + " "
            tcmd = PYTHON + ' run.py ' + c + t
            print('\t\t<Test name="%s" cmd="%s" wdir="tests/pjsua" />' % (tname, tcmd))
        sys.exit(0)
    elif sys.argv[0] == '-s' or sys.argv[0] == '--shell':
        if len(sys.argv) > 1:
            shell_cmd = sys.argv[1]
            sys.argv.pop(0)
            sys.argv.pop(0)
        else:
            sys.argv.pop(0)
            sys.stderr.write("Error: argument value required")
            sys.exit(1)
    elif sys.argv[0] == '-d' or sys.argv[0] == '--disable':
        if len(sys.argv) > 1:
            excluded_tests.append(sys.argv[1])
            sys.argv.pop(0)
            sys.argv.pop(0)
        else:
            sys.argv.pop(0)
            sys.stderr.write("Error: argument value required")
            sys.exit(1)
    elif sys.argv[0] == '--no-log':
        sys.argv.pop(0)
        with_log=False
    else:
        # should be run.py options
        break


# Generate arguments for run.py
argv_st = " ".join(sys.argv) + " "

# Init vars
fails_cnt = 0
tests_cnt = 0

# Filter-out excluded tests
for pat in excluded_tests:
    tests = [t for t in tests if t.find(pat)==-1]

# Now run the tests
total_cnt = len(tests)
for t in tests:
    if resume_script!="" and t.find(resume_script)==-1:
        print("Skipping " + t +"..")
        total_cnt = total_cnt - 1
        continue
    resume_script=""
    cmdline = "python run.py " + argv_st + t
    if shell_cmd:
        cmdline = "%s '%s'" % (shell_cmd, cmdline)
    for i in range(1, retry_num+2):
        t0 = time.time()
        msg = "Running %3d/%d (#%d): %s..." % (tests_cnt+1, total_cnt, i, cmdline)
        sys.stdout.write(msg)
        sys.stdout.flush()
        if with_log:
            logname = re.search(".*\s+(.*)", t).group(1)
            logname = re.sub("[\\\/]", "_", logname)
            logname = re.sub("\.py$", ".log", logname)
            logname = re.sub("\.xml$", ".log", logname)
            logname = "logs/" + logname
        else:
            logname = os.devnull
        ret = os.system(cmdline + " > " + logname)
        t1 = time.time()
        if ret != 0:
            dur = int(t1 - t0)
            print(" failed!! [" + str(dur) + "s]")
            if (i < retry_num + 1):
                continue
            if with_log:
                lines = open(logname, "r", encoding='utf-8').readlines()
                print(''.join(lines))
                print("Log file: '" + logname + "'.")
            fails_cnt += 1
        else:
            dur = int(t1 - t0)
            print(" ok [" + str(dur) + "s]")
            break
    tests_cnt += 1

if fails_cnt == 0:
    print("All " + str(tests_cnt) + " tests completed successfully")
else:
    print(str(tests_cnt) + " tests completed, " +  str(fails_cnt) + " test(s) failed")

sys.exit(fails_cnt)