profile.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611
  1. #! /usr/bin/env python3
  2. #
  3. # Class for profiling python code. rev 1.0 6/2/94
  4. #
  5. # Written by James Roskind
  6. # Based on prior profile module by Sjoerd Mullender...
  7. # which was hacked somewhat by: Guido van Rossum
  8. """Class for profiling Python code."""
  9. # Copyright Disney Enterprises, Inc. All Rights Reserved.
  10. # Licensed to PSF under a Contributor Agreement
  11. #
  12. # Licensed under the Apache License, Version 2.0 (the "License");
  13. # you may not use this file except in compliance with the License.
  14. # You may obtain a copy of the License at
  15. #
  16. # http://www.apache.org/licenses/LICENSE-2.0
  17. #
  18. # Unless required by applicable law or agreed to in writing, software
  19. # distributed under the License is distributed on an "AS IS" BASIS,
  20. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
  21. # either express or implied. See the License for the specific language
  22. # governing permissions and limitations under the License.
  23. import io
  24. import sys
  25. import time
  26. import marshal
  27. __all__ = ["run", "runctx", "Profile"]
  28. # Sample timer for use with
  29. #i_count = 0
  30. #def integer_timer():
  31. # global i_count
  32. # i_count = i_count + 1
  33. # return i_count
  34. #itimes = integer_timer # replace with C coded timer returning integers
  35. class _Utils:
  36. """Support class for utility functions which are shared by
  37. profile.py and cProfile.py modules.
  38. Not supposed to be used directly.
  39. """
  40. def __init__(self, profiler):
  41. self.profiler = profiler
  42. def run(self, statement, filename, sort):
  43. prof = self.profiler()
  44. try:
  45. prof.run(statement)
  46. except SystemExit:
  47. pass
  48. finally:
  49. self._show(prof, filename, sort)
  50. def runctx(self, statement, globals, locals, filename, sort):
  51. prof = self.profiler()
  52. try:
  53. prof.runctx(statement, globals, locals)
  54. except SystemExit:
  55. pass
  56. finally:
  57. self._show(prof, filename, sort)
  58. def _show(self, prof, filename, sort):
  59. if filename is not None:
  60. prof.dump_stats(filename)
  61. else:
  62. prof.print_stats(sort)
  63. #**************************************************************************
  64. # The following are the static member functions for the profiler class
  65. # Note that an instance of Profile() is *not* needed to call them.
  66. #**************************************************************************
  67. def run(statement, filename=None, sort=-1):
  68. """Run statement under profiler optionally saving results in filename
  69. This function takes a single argument that can be passed to the
  70. "exec" statement, and an optional file name. In all cases this
  71. routine attempts to "exec" its first argument and gather profiling
  72. statistics from the execution. If no file name is present, then this
  73. function automatically prints a simple profiling report, sorted by the
  74. standard name string (file/line/function-name) that is presented in
  75. each line.
  76. """
  77. return _Utils(Profile).run(statement, filename, sort)
  78. def runctx(statement, globals, locals, filename=None, sort=-1):
  79. """Run statement under profiler, supplying your own globals and locals,
  80. optionally saving results in filename.
  81. statement and filename have the same semantics as profile.run
  82. """
  83. return _Utils(Profile).runctx(statement, globals, locals, filename, sort)
  84. class Profile:
  85. """Profiler class.
  86. self.cur is always a tuple. Each such tuple corresponds to a stack
  87. frame that is currently active (self.cur[-2]). The following are the
  88. definitions of its members. We use this external "parallel stack" to
  89. avoid contaminating the program that we are profiling. (old profiler
  90. used to write into the frames local dictionary!!) Derived classes
  91. can change the definition of some entries, as long as they leave
  92. [-2:] intact (frame and previous tuple). In case an internal error is
  93. detected, the -3 element is used as the function name.
  94. [ 0] = Time that needs to be charged to the parent frame's function.
  95. It is used so that a function call will not have to access the
  96. timing data for the parent frame.
  97. [ 1] = Total time spent in this frame's function, excluding time in
  98. subfunctions (this latter is tallied in cur[2]).
  99. [ 2] = Total time spent in subfunctions, excluding time executing the
  100. frame's function (this latter is tallied in cur[1]).
  101. [-3] = Name of the function that corresponds to this frame.
  102. [-2] = Actual frame that we correspond to (used to sync exception handling).
  103. [-1] = Our parent 6-tuple (corresponds to frame.f_back).
  104. Timing data for each function is stored as a 5-tuple in the dictionary
  105. self.timings[]. The index is always the name stored in self.cur[-3].
  106. The following are the definitions of the members:
  107. [0] = The number of times this function was called, not counting direct
  108. or indirect recursion,
  109. [1] = Number of times this function appears on the stack, minus one
  110. [2] = Total time spent internal to this function
  111. [3] = Cumulative time that this function was present on the stack. In
  112. non-recursive functions, this is the total execution time from start
  113. to finish of each invocation of a function, including time spent in
  114. all subfunctions.
  115. [4] = A dictionary indicating for each function name, the number of times
  116. it was called by us.
  117. """
  118. bias = 0 # calibration constant
  119. def __init__(self, timer=None, bias=None):
  120. self.timings = {}
  121. self.cur = None
  122. self.cmd = ""
  123. self.c_func_name = ""
  124. if bias is None:
  125. bias = self.bias
  126. self.bias = bias # Materialize in local dict for lookup speed.
  127. if not timer:
  128. self.timer = self.get_time = time.process_time
  129. self.dispatcher = self.trace_dispatch_i
  130. else:
  131. self.timer = timer
  132. t = self.timer() # test out timer function
  133. try:
  134. length = len(t)
  135. except TypeError:
  136. self.get_time = timer
  137. self.dispatcher = self.trace_dispatch_i
  138. else:
  139. if length == 2:
  140. self.dispatcher = self.trace_dispatch
  141. else:
  142. self.dispatcher = self.trace_dispatch_l
  143. # This get_time() implementation needs to be defined
  144. # here to capture the passed-in timer in the parameter
  145. # list (for performance). Note that we can't assume
  146. # the timer() result contains two values in all
  147. # cases.
  148. def get_time_timer(timer=timer, sum=sum):
  149. return sum(timer())
  150. self.get_time = get_time_timer
  151. self.t = self.get_time()
  152. self.simulate_call('profiler')
  153. # Heavily optimized dispatch routine for time.process_time() timer
  154. def trace_dispatch(self, frame, event, arg):
  155. timer = self.timer
  156. t = timer()
  157. t = t[0] + t[1] - self.t - self.bias
  158. if event == "c_call":
  159. self.c_func_name = arg.__name__
  160. if self.dispatch[event](self, frame,t):
  161. t = timer()
  162. self.t = t[0] + t[1]
  163. else:
  164. r = timer()
  165. self.t = r[0] + r[1] - t # put back unrecorded delta
  166. # Dispatch routine for best timer program (return = scalar, fastest if
  167. # an integer but float works too -- and time.process_time() relies on that).
  168. def trace_dispatch_i(self, frame, event, arg):
  169. timer = self.timer
  170. t = timer() - self.t - self.bias
  171. if event == "c_call":
  172. self.c_func_name = arg.__name__
  173. if self.dispatch[event](self, frame, t):
  174. self.t = timer()
  175. else:
  176. self.t = timer() - t # put back unrecorded delta
  177. # Dispatch routine for macintosh (timer returns time in ticks of
  178. # 1/60th second)
  179. def trace_dispatch_mac(self, frame, event, arg):
  180. timer = self.timer
  181. t = timer()/60.0 - self.t - self.bias
  182. if event == "c_call":
  183. self.c_func_name = arg.__name__
  184. if self.dispatch[event](self, frame, t):
  185. self.t = timer()/60.0
  186. else:
  187. self.t = timer()/60.0 - t # put back unrecorded delta
  188. # SLOW generic dispatch routine for timer returning lists of numbers
  189. def trace_dispatch_l(self, frame, event, arg):
  190. get_time = self.get_time
  191. t = get_time() - self.t - self.bias
  192. if event == "c_call":
  193. self.c_func_name = arg.__name__
  194. if self.dispatch[event](self, frame, t):
  195. self.t = get_time()
  196. else:
  197. self.t = get_time() - t # put back unrecorded delta
  198. # In the event handlers, the first 3 elements of self.cur are unpacked
  199. # into vrbls w/ 3-letter names. The last two characters are meant to be
  200. # mnemonic:
  201. # _pt self.cur[0] "parent time" time to be charged to parent frame
  202. # _it self.cur[1] "internal time" time spent directly in the function
  203. # _et self.cur[2] "external time" time spent in subfunctions
  204. def trace_dispatch_exception(self, frame, t):
  205. rpt, rit, ret, rfn, rframe, rcur = self.cur
  206. if (rframe is not frame) and rcur:
  207. return self.trace_dispatch_return(rframe, t)
  208. self.cur = rpt, rit+t, ret, rfn, rframe, rcur
  209. return 1
  210. def trace_dispatch_call(self, frame, t):
  211. if self.cur and frame.f_back is not self.cur[-2]:
  212. rpt, rit, ret, rfn, rframe, rcur = self.cur
  213. if not isinstance(rframe, Profile.fake_frame):
  214. assert rframe.f_back is frame.f_back, ("Bad call", rfn,
  215. rframe, rframe.f_back,
  216. frame, frame.f_back)
  217. self.trace_dispatch_return(rframe, 0)
  218. assert (self.cur is None or \
  219. frame.f_back is self.cur[-2]), ("Bad call",
  220. self.cur[-3])
  221. fcode = frame.f_code
  222. fn = (fcode.co_filename, fcode.co_firstlineno, fcode.co_name)
  223. self.cur = (t, 0, 0, fn, frame, self.cur)
  224. timings = self.timings
  225. if fn in timings:
  226. cc, ns, tt, ct, callers = timings[fn]
  227. timings[fn] = cc, ns + 1, tt, ct, callers
  228. else:
  229. timings[fn] = 0, 0, 0, 0, {}
  230. return 1
  231. def trace_dispatch_c_call (self, frame, t):
  232. fn = ("", 0, self.c_func_name)
  233. self.cur = (t, 0, 0, fn, frame, self.cur)
  234. timings = self.timings
  235. if fn in timings:
  236. cc, ns, tt, ct, callers = timings[fn]
  237. timings[fn] = cc, ns+1, tt, ct, callers
  238. else:
  239. timings[fn] = 0, 0, 0, 0, {}
  240. return 1
  241. def trace_dispatch_return(self, frame, t):
  242. if frame is not self.cur[-2]:
  243. assert frame is self.cur[-2].f_back, ("Bad return", self.cur[-3])
  244. self.trace_dispatch_return(self.cur[-2], 0)
  245. # Prefix "r" means part of the Returning or exiting frame.
  246. # Prefix "p" means part of the Previous or Parent or older frame.
  247. rpt, rit, ret, rfn, frame, rcur = self.cur
  248. rit = rit + t
  249. frame_total = rit + ret
  250. ppt, pit, pet, pfn, pframe, pcur = rcur
  251. self.cur = ppt, pit + rpt, pet + frame_total, pfn, pframe, pcur
  252. timings = self.timings
  253. cc, ns, tt, ct, callers = timings[rfn]
  254. if not ns:
  255. # This is the only occurrence of the function on the stack.
  256. # Else this is a (directly or indirectly) recursive call, and
  257. # its cumulative time will get updated when the topmost call to
  258. # it returns.
  259. ct = ct + frame_total
  260. cc = cc + 1
  261. if pfn in callers:
  262. callers[pfn] = callers[pfn] + 1 # hack: gather more
  263. # stats such as the amount of time added to ct courtesy
  264. # of this specific call, and the contribution to cc
  265. # courtesy of this call.
  266. else:
  267. callers[pfn] = 1
  268. timings[rfn] = cc, ns - 1, tt + rit, ct, callers
  269. return 1
  270. dispatch = {
  271. "call": trace_dispatch_call,
  272. "exception": trace_dispatch_exception,
  273. "return": trace_dispatch_return,
  274. "c_call": trace_dispatch_c_call,
  275. "c_exception": trace_dispatch_return, # the C function returned
  276. "c_return": trace_dispatch_return,
  277. }
  278. # The next few functions play with self.cmd. By carefully preloading
  279. # our parallel stack, we can force the profiled result to include
  280. # an arbitrary string as the name of the calling function.
  281. # We use self.cmd as that string, and the resulting stats look
  282. # very nice :-).
  283. def set_cmd(self, cmd):
  284. if self.cur[-1]: return # already set
  285. self.cmd = cmd
  286. self.simulate_call(cmd)
  287. class fake_code:
  288. def __init__(self, filename, line, name):
  289. self.co_filename = filename
  290. self.co_line = line
  291. self.co_name = name
  292. self.co_firstlineno = 0
  293. def __repr__(self):
  294. return repr((self.co_filename, self.co_line, self.co_name))
  295. class fake_frame:
  296. def __init__(self, code, prior):
  297. self.f_code = code
  298. self.f_back = prior
  299. def simulate_call(self, name):
  300. code = self.fake_code('profile', 0, name)
  301. if self.cur:
  302. pframe = self.cur[-2]
  303. else:
  304. pframe = None
  305. frame = self.fake_frame(code, pframe)
  306. self.dispatch['call'](self, frame, 0)
  307. # collect stats from pending stack, including getting final
  308. # timings for self.cmd frame.
  309. def simulate_cmd_complete(self):
  310. get_time = self.get_time
  311. t = get_time() - self.t
  312. while self.cur[-1]:
  313. # We *can* cause assertion errors here if
  314. # dispatch_trace_return checks for a frame match!
  315. self.dispatch['return'](self, self.cur[-2], t)
  316. t = 0
  317. self.t = get_time() - t
  318. def print_stats(self, sort=-1):
  319. import pstats
  320. pstats.Stats(self).strip_dirs().sort_stats(sort). \
  321. print_stats()
  322. def dump_stats(self, file):
  323. with open(file, 'wb') as f:
  324. self.create_stats()
  325. marshal.dump(self.stats, f)
  326. def create_stats(self):
  327. self.simulate_cmd_complete()
  328. self.snapshot_stats()
  329. def snapshot_stats(self):
  330. self.stats = {}
  331. for func, (cc, ns, tt, ct, callers) in self.timings.items():
  332. callers = callers.copy()
  333. nc = 0
  334. for callcnt in callers.values():
  335. nc += callcnt
  336. self.stats[func] = cc, nc, tt, ct, callers
  337. # The following two methods can be called by clients to use
  338. # a profiler to profile a statement, given as a string.
  339. def run(self, cmd):
  340. import __main__
  341. dict = __main__.__dict__
  342. return self.runctx(cmd, dict, dict)
  343. def runctx(self, cmd, globals, locals):
  344. self.set_cmd(cmd)
  345. sys.setprofile(self.dispatcher)
  346. try:
  347. exec(cmd, globals, locals)
  348. finally:
  349. sys.setprofile(None)
  350. return self
  351. # This method is more useful to profile a single function call.
  352. def runcall(self, func, /, *args, **kw):
  353. self.set_cmd(repr(func))
  354. sys.setprofile(self.dispatcher)
  355. try:
  356. return func(*args, **kw)
  357. finally:
  358. sys.setprofile(None)
  359. #******************************************************************
  360. # The following calculates the overhead for using a profiler. The
  361. # problem is that it takes a fair amount of time for the profiler
  362. # to stop the stopwatch (from the time it receives an event).
  363. # Similarly, there is a delay from the time that the profiler
  364. # re-starts the stopwatch before the user's code really gets to
  365. # continue. The following code tries to measure the difference on
  366. # a per-event basis.
  367. #
  368. # Note that this difference is only significant if there are a lot of
  369. # events, and relatively little user code per event. For example,
  370. # code with small functions will typically benefit from having the
  371. # profiler calibrated for the current platform. This *could* be
  372. # done on the fly during init() time, but it is not worth the
  373. # effort. Also note that if too large a value specified, then
  374. # execution time on some functions will actually appear as a
  375. # negative number. It is *normal* for some functions (with very
  376. # low call counts) to have such negative stats, even if the
  377. # calibration figure is "correct."
  378. #
  379. # One alternative to profile-time calibration adjustments (i.e.,
  380. # adding in the magic little delta during each event) is to track
  381. # more carefully the number of events (and cumulatively, the number
  382. # of events during sub functions) that are seen. If this were
  383. # done, then the arithmetic could be done after the fact (i.e., at
  384. # display time). Currently, we track only call/return events.
  385. # These values can be deduced by examining the callees and callers
  386. # vectors for each functions. Hence we *can* almost correct the
  387. # internal time figure at print time (note that we currently don't
  388. # track exception event processing counts). Unfortunately, there
  389. # is currently no similar information for cumulative sub-function
  390. # time. It would not be hard to "get all this info" at profiler
  391. # time. Specifically, we would have to extend the tuples to keep
  392. # counts of this in each frame, and then extend the defs of timing
  393. # tuples to include the significant two figures. I'm a bit fearful
  394. # that this additional feature will slow the heavily optimized
  395. # event/time ratio (i.e., the profiler would run slower, fur a very
  396. # low "value added" feature.)
  397. #**************************************************************
  398. def calibrate(self, m, verbose=0):
  399. if self.__class__ is not Profile:
  400. raise TypeError("Subclasses must override .calibrate().")
  401. saved_bias = self.bias
  402. self.bias = 0
  403. try:
  404. return self._calibrate_inner(m, verbose)
  405. finally:
  406. self.bias = saved_bias
  407. def _calibrate_inner(self, m, verbose):
  408. get_time = self.get_time
  409. # Set up a test case to be run with and without profiling. Include
  410. # lots of calls, because we're trying to quantify stopwatch overhead.
  411. # Do not raise any exceptions, though, because we want to know
  412. # exactly how many profile events are generated (one call event, +
  413. # one return event, per Python-level call).
  414. def f1(n):
  415. for i in range(n):
  416. x = 1
  417. def f(m, f1=f1):
  418. for i in range(m):
  419. f1(100)
  420. f(m) # warm up the cache
  421. # elapsed_noprofile <- time f(m) takes without profiling.
  422. t0 = get_time()
  423. f(m)
  424. t1 = get_time()
  425. elapsed_noprofile = t1 - t0
  426. if verbose:
  427. print("elapsed time without profiling =", elapsed_noprofile)
  428. # elapsed_profile <- time f(m) takes with profiling. The difference
  429. # is profiling overhead, only some of which the profiler subtracts
  430. # out on its own.
  431. p = Profile()
  432. t0 = get_time()
  433. p.runctx('f(m)', globals(), locals())
  434. t1 = get_time()
  435. elapsed_profile = t1 - t0
  436. if verbose:
  437. print("elapsed time with profiling =", elapsed_profile)
  438. # reported_time <- "CPU seconds" the profiler charged to f and f1.
  439. total_calls = 0.0
  440. reported_time = 0.0
  441. for (filename, line, funcname), (cc, ns, tt, ct, callers) in \
  442. p.timings.items():
  443. if funcname in ("f", "f1"):
  444. total_calls += cc
  445. reported_time += tt
  446. if verbose:
  447. print("'CPU seconds' profiler reported =", reported_time)
  448. print("total # calls =", total_calls)
  449. if total_calls != m + 1:
  450. raise ValueError("internal error: total calls = %d" % total_calls)
  451. # reported_time - elapsed_noprofile = overhead the profiler wasn't
  452. # able to measure. Divide by twice the number of calls (since there
  453. # are two profiler events per call in this test) to get the hidden
  454. # overhead per event.
  455. mean = (reported_time - elapsed_noprofile) / 2.0 / total_calls
  456. if verbose:
  457. print("mean stopwatch overhead per profile event =", mean)
  458. return mean
  459. #****************************************************************************
  460. def main():
  461. import os
  462. from optparse import OptionParser
  463. usage = "profile.py [-o output_file_path] [-s sort] [-m module | scriptfile] [arg] ..."
  464. parser = OptionParser(usage=usage)
  465. parser.allow_interspersed_args = False
  466. parser.add_option('-o', '--outfile', dest="outfile",
  467. help="Save stats to <outfile>", default=None)
  468. parser.add_option('-m', dest="module", action="store_true",
  469. help="Profile a library module.", default=False)
  470. parser.add_option('-s', '--sort', dest="sort",
  471. help="Sort order when printing to stdout, based on pstats.Stats class",
  472. default=-1)
  473. if not sys.argv[1:]:
  474. parser.print_usage()
  475. sys.exit(2)
  476. (options, args) = parser.parse_args()
  477. sys.argv[:] = args
  478. # The script that we're profiling may chdir, so capture the absolute path
  479. # to the output file at startup.
  480. if options.outfile is not None:
  481. options.outfile = os.path.abspath(options.outfile)
  482. if len(args) > 0:
  483. if options.module:
  484. import runpy
  485. code = "run_module(modname, run_name='__main__')"
  486. globs = {
  487. 'run_module': runpy.run_module,
  488. 'modname': args[0]
  489. }
  490. else:
  491. progname = args[0]
  492. sys.path.insert(0, os.path.dirname(progname))
  493. with io.open_code(progname) as fp:
  494. code = compile(fp.read(), progname, 'exec')
  495. globs = {
  496. '__file__': progname,
  497. '__name__': '__main__',
  498. '__package__': None,
  499. '__cached__': None,
  500. }
  501. try:
  502. runctx(code, globs, None, options.outfile, options.sort)
  503. except BrokenPipeError as exc:
  504. # Prevent "Exception ignored" during interpreter shutdown.
  505. sys.stdout = None
  506. sys.exit(exc.errno)
  507. else:
  508. parser.print_usage()
  509. return parser
  510. # When invoked as main program, invoke the profiler on a script
  511. if __name__ == '__main__':
  512. main()