ecoutils.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521
  1. # -*- coding: utf-8 -*-
  2. # Copyright (c) 2013, Mahmoud Hashemi
  3. #
  4. # Redistribution and use in source and binary forms, with or without
  5. # modification, are permitted provided that the following conditions are
  6. # met:
  7. #
  8. # * Redistributions of source code must retain the above copyright
  9. # notice, this list of conditions and the following disclaimer.
  10. #
  11. # * Redistributions in binary form must reproduce the above
  12. # copyright notice, this list of conditions and the following
  13. # disclaimer in the documentation and/or other materials provided
  14. # with the distribution.
  15. #
  16. # * The names of the contributors may not be used to endorse or
  17. # promote products derived from this software without specific
  18. # prior written permission.
  19. #
  20. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31. """As a programming ecosystem grows, so do the chances of runtime
  32. variability.
  33. Python boasts one of the widest deployments for a high-level
  34. programming environment, making it a viable target for all manner of
  35. application. But with breadth comes variance, so it's important to
  36. know what you're working with.
  37. Some basic variations that are common among development machines:
  38. * **Executable runtime**: CPython, PyPy, Jython, etc., plus build date and compiler
  39. * **Language version**: 2.4, 2.5, 2.6, 2.7... 3.4, 3.5, 3.6
  40. * **Host operating system**: Windows, OS X, Ubuntu, Debian, CentOS, RHEL, etc.
  41. * **Features**: 64-bit, IPv6, Unicode character support (UCS-2/UCS-4)
  42. * **Built-in library support**: OpenSSL, threading, SQLite, zlib
  43. * **User environment**: umask, ulimit, working directory path
  44. * **Machine info**: CPU count, hostname, filesystem encoding
  45. See the full example profile below for more.
  46. ecoutils was created to quantify that variability. ecoutils quickly
  47. produces an information-dense description of critical runtime factors,
  48. with minimal side effects. In short, ecoutils is like browser and user
  49. agent analytics, but for Python environments.
  50. Transmission and collection
  51. ---------------------------
  52. The data is all JSON serializable, and is suitable for sending to a
  53. central analytics server. An HTTP-backed service for this can be found
  54. at: https://github.com/mahmoud/espymetrics/
  55. Notable omissions
  56. -----------------
  57. Due to space constraints (and possibly latency constraints), the
  58. following information is deemed not dense enough, and thus omitted:
  59. * :data:`sys.path`
  60. * full :mod:`sysconfig`
  61. * environment variables (:data:`os.environ`)
  62. Compatibility
  63. -------------
  64. So far ecoutils has has been tested on Python 2.4, 2.5, 2.6, 2.7, 3.4,
  65. 3.5, and PyPy. Various versions have been tested on Ubuntu, Debian,
  66. RHEL, OS X, FreeBSD, and Windows 7.
  67. .. note:: Boltons typically only support back to Python 2.6, but due
  68. to its nature, ecoutils extends backwards compatibility to Python
  69. 2.4 and 2.5.
  70. Profile generation
  71. ------------------
  72. Profiles are generated by :func:`ecoutils.get_profile`.
  73. When run as a module, ecoutils will call :func:`~ecoutils.get_profile`
  74. and print a profile in JSON format::
  75. $ python -m boltons.ecoutils
  76. {
  77. "_eco_version": "1.0.0",
  78. "cpu_count": 4,
  79. "cwd": "/home/mahmoud/projects/boltons",
  80. "fs_encoding": "UTF-8",
  81. "guid": "6b139e7bbf5ad4ed8d4063bf6235b4d2",
  82. "hostfqdn": "mahmoud-host",
  83. "hostname": "mahmoud-host",
  84. "linux_dist_name": "Ubuntu",
  85. "linux_dist_version": "14.04",
  86. "python": {
  87. "argv": "boltons/ecoutils.py",
  88. "bin": "/usr/bin/python",
  89. "build_date": "Jun 22 2015 17:58:13",
  90. "compiler": "GCC 4.8.2",
  91. "features": {
  92. "64bit": true,
  93. "expat": "expat_2.1.0",
  94. "ipv6": true,
  95. "openssl": "OpenSSL 1.0.1f 6 Jan 2014",
  96. "readline": true,
  97. "sqlite": "3.8.2",
  98. "threading": true,
  99. "tkinter": "8.6",
  100. "unicode_wide": true,
  101. "zlib": "1.2.8"
  102. },
  103. "version": "2.7.6 (default, Jun 22 2015, 17:58:13) [GCC 4.8.2]",
  104. "version_info": [
  105. 2,
  106. 7,
  107. 6,
  108. "final",
  109. 0
  110. ]
  111. },
  112. "time_utc": "2016-05-24 07:59:40.473140",
  113. "time_utc_offset": -8.0,
  114. "ulimit_hard": 4096,
  115. "ulimit_soft": 1024,
  116. "umask": "002",
  117. "uname": {
  118. "machine": "x86_64",
  119. "node": "mahmoud-host",
  120. "processor": "x86_64",
  121. "release": "3.13.0-85-generic",
  122. "system": "Linux",
  123. "version": "#129-Ubuntu SMP Thu Mar 17 20:50:15 UTC 2016"
  124. },
  125. "username": "mahmoud"
  126. }
  127. ``pip install boltons`` and try it yourself!
  128. """
  129. import re
  130. import os
  131. import sys
  132. import time
  133. import pprint
  134. import random
  135. import socket
  136. import struct
  137. import getpass
  138. import datetime
  139. import platform
  140. ECO_VERSION = '1.0.1' # see version history below
  141. PY_GT_2 = sys.version_info[0] > 2
  142. try:
  143. getrandbits = random.SystemRandom().getrandbits
  144. HAVE_URANDOM = True
  145. except Exception:
  146. HAVE_URANDOM = False
  147. getrandbits = random.getrandbits
  148. # 128-bit GUID just like a UUID, but backwards compatible to 2.4
  149. INSTANCE_ID = hex(getrandbits(128))[2:-1].lower()
  150. IS_64BIT = struct.calcsize("P") > 4
  151. HAVE_UCS4 = getattr(sys, 'maxunicode', 0) > 65536
  152. HAVE_READLINE = True
  153. try:
  154. import readline
  155. except Exception:
  156. HAVE_READLINE = False
  157. try:
  158. import sqlite3
  159. SQLITE_VERSION = sqlite3.sqlite_version
  160. except Exception:
  161. # note: 2.5 and older have sqlite, but not sqlite3
  162. SQLITE_VERSION = ''
  163. try:
  164. import ssl
  165. try:
  166. OPENSSL_VERSION = ssl.OPENSSL_VERSION
  167. except AttributeError:
  168. # This is a conservative estimate for Python <2.6
  169. # SSL module added in 2006, when 0.9.7 was standard
  170. OPENSSL_VERSION = 'OpenSSL >0.8.0'
  171. except Exception:
  172. OPENSSL_VERSION = ''
  173. try:
  174. if PY_GT_2:
  175. import tkinter
  176. else:
  177. import Tkinter as tkinter
  178. TKINTER_VERSION = str(tkinter.TkVersion)
  179. except Exception:
  180. TKINTER_VERSION = ''
  181. try:
  182. import zlib
  183. ZLIB_VERSION = zlib.ZLIB_VERSION
  184. except Exception:
  185. ZLIB_VERSION = ''
  186. try:
  187. from xml.parsers import expat
  188. EXPAT_VERSION = expat.EXPAT_VERSION
  189. except Exception:
  190. EXPAT_VERSION = ''
  191. try:
  192. from multiprocessing import cpu_count
  193. CPU_COUNT = cpu_count()
  194. except Exception:
  195. CPU_COUNT = 0
  196. try:
  197. import threading
  198. HAVE_THREADING = True
  199. except Exception:
  200. HAVE_THREADING = False
  201. try:
  202. HAVE_IPV6 = socket.has_ipv6
  203. except Exception:
  204. HAVE_IPV6 = False
  205. try:
  206. from resource import getrlimit, RLIMIT_NOFILE
  207. RLIMIT_FDS_SOFT, RLIMIT_FDS_HARD = getrlimit(RLIMIT_NOFILE)
  208. except Exception:
  209. RLIMIT_FDS_SOFT, RLIMIT_FDS_HARD = 0, 0
  210. START_TIME_INFO = {'time_utc': str(datetime.datetime.utcnow()),
  211. 'time_utc_offset': -time.timezone / 3600.0}
  212. def get_python_info():
  213. ret = {}
  214. ret['argv'] = _escape_shell_args(sys.argv)
  215. ret['bin'] = sys.executable
  216. # Even though compiler/build_date are already here, they're
  217. # actually parsed from the version string. So, in the rare case of
  218. # the unparsable version string, we're still transmitting it.
  219. ret['version'] = ' '.join(sys.version.split())
  220. ret['compiler'] = platform.python_compiler()
  221. ret['build_date'] = platform.python_build()[1]
  222. ret['version_info'] = list(sys.version_info)
  223. ret['features'] = {'openssl': OPENSSL_VERSION,
  224. 'expat': EXPAT_VERSION,
  225. 'sqlite': SQLITE_VERSION,
  226. 'tkinter': TKINTER_VERSION,
  227. 'zlib': ZLIB_VERSION,
  228. 'unicode_wide': HAVE_UCS4,
  229. 'readline': HAVE_READLINE,
  230. '64bit': IS_64BIT,
  231. 'ipv6': HAVE_IPV6,
  232. 'threading': HAVE_THREADING,
  233. 'urandom': HAVE_URANDOM}
  234. return ret
  235. def get_profile(**kwargs):
  236. """The main entrypoint to ecoutils. Calling this will return a
  237. JSON-serializable dictionary of information about the current
  238. process.
  239. It is very unlikely that the information returned will change
  240. during the lifetime of the process, and in most cases the majority
  241. of the information stays the same between runs as well.
  242. :func:`get_profile` takes one optional keyword argument, *scrub*,
  243. a :class:`bool` that, if True, blanks out identifiable
  244. information. This includes current working directory, hostname,
  245. Python executable path, command-line arguments, and
  246. username. Values are replaced with '-', but for compatibility keys
  247. remain in place.
  248. """
  249. scrub = kwargs.pop('scrub', False)
  250. if kwargs:
  251. raise TypeError('unexpected keyword arguments: %r' % (kwargs.keys(),))
  252. ret = {}
  253. try:
  254. ret['username'] = getpass.getuser()
  255. except Exception:
  256. ret['username'] = ''
  257. ret['guid'] = str(INSTANCE_ID)
  258. ret['hostname'] = socket.gethostname()
  259. ret['hostfqdn'] = socket.getfqdn()
  260. uname = platform.uname()
  261. ret['uname'] = {'system': uname[0],
  262. 'node': uname[1],
  263. 'release': uname[2], # linux: distro name
  264. 'version': uname[3], # linux: kernel version
  265. 'machine': uname[4],
  266. 'processor': uname[5]}
  267. try:
  268. linux_dist = platform.linux_distribution()
  269. except Exception:
  270. linux_dist = ('', '', '')
  271. ret['linux_dist_name'] = linux_dist[0]
  272. ret['linux_dist_version'] = linux_dist[1]
  273. ret['cpu_count'] = CPU_COUNT
  274. ret['fs_encoding'] = sys.getfilesystemencoding()
  275. ret['ulimit_soft'] = RLIMIT_FDS_SOFT
  276. ret['ulimit_hard'] = RLIMIT_FDS_HARD
  277. ret['cwd'] = os.getcwd()
  278. ret['umask'] = oct(os.umask(os.umask(2))).rjust(3, '0')
  279. ret['python'] = get_python_info()
  280. ret.update(START_TIME_INFO)
  281. ret['_eco_version'] = ECO_VERSION
  282. if scrub:
  283. # mask identifiable information
  284. ret['cwd'] = '-'
  285. ret['hostname'] = '-'
  286. ret['hostfqdn'] = '-'
  287. ret['python']['bin'] = '-'
  288. ret['python']['argv'] = '-'
  289. ret['uname']['node'] = '-'
  290. ret['username'] = '-'
  291. return ret
  292. try:
  293. import json
  294. def dumps(val, indent):
  295. if indent:
  296. return json.dumps(val, sort_keys=True, indent=indent)
  297. return json.dumps(val, sort_keys=True)
  298. except ImportError:
  299. _real_safe_repr = pprint._safe_repr
  300. def _fake_json_dumps(val, indent=2):
  301. # never do this. this is a hack for Python 2.4. Python 2.5 added
  302. # the json module for a reason.
  303. def _fake_safe_repr(*a, **kw):
  304. res, is_read, is_rec = _real_safe_repr(*a, **kw)
  305. if res == 'None':
  306. res = 'null'
  307. if res == 'True':
  308. res = 'true'
  309. if res == 'False':
  310. res = 'false'
  311. if not (res.startswith("'") or res.startswith("u'")):
  312. res = res
  313. else:
  314. if res.startswith('u'):
  315. res = res[1:]
  316. contents = res[1:-1]
  317. contents = contents.replace('"', '').replace(r'\"', '')
  318. res = '"' + contents + '"'
  319. return res, is_read, is_rec
  320. pprint._safe_repr = _fake_safe_repr
  321. try:
  322. ret = pprint.pformat(val, indent=indent)
  323. finally:
  324. pprint._safe_repr = _real_safe_repr
  325. return ret
  326. def dumps(val, indent):
  327. ret = _fake_json_dumps(val, indent=indent)
  328. if not indent:
  329. ret = re.sub(r'\n\s*', ' ', ret)
  330. return ret
  331. def get_profile_json(indent=False):
  332. if indent:
  333. indent = 2
  334. else:
  335. indent = 0
  336. data_dict = get_profile()
  337. return dumps(data_dict, indent)
  338. def main():
  339. print(get_profile_json(indent=True))
  340. #############################################
  341. # The shell escaping copied in from strutils
  342. #############################################
  343. def _escape_shell_args(args, sep=' ', style=None):
  344. if not style:
  345. if sys.platform == 'win32':
  346. style = 'cmd'
  347. else:
  348. style = 'sh'
  349. if style == 'sh':
  350. return _args2sh(args, sep=sep)
  351. elif style == 'cmd':
  352. return _args2cmd(args, sep=sep)
  353. raise ValueError("style expected one of 'cmd' or 'sh', not %r" % style)
  354. _find_sh_unsafe = re.compile(r'[^a-zA-Z0-9_@%+=:,./-]').search
  355. def _args2sh(args, sep=' '):
  356. # see strutils
  357. ret_list = []
  358. for arg in args:
  359. if not arg:
  360. ret_list.append("''")
  361. continue
  362. if _find_sh_unsafe(arg) is None:
  363. ret_list.append(arg)
  364. continue
  365. # use single quotes, and put single quotes into double quotes
  366. # the string $'b is then quoted as '$'"'"'b'
  367. ret_list.append("'" + arg.replace("'", "'\"'\"'") + "'")
  368. return ' '.join(ret_list)
  369. def _args2cmd(args, sep=' '):
  370. # see strutils
  371. result = []
  372. needquote = False
  373. for arg in args:
  374. bs_buf = []
  375. # Add a space to separate this argument from the others
  376. if result:
  377. result.append(' ')
  378. needquote = (" " in arg) or ("\t" in arg) or not arg
  379. if needquote:
  380. result.append('"')
  381. for c in arg:
  382. if c == '\\':
  383. # Don't know if we need to double yet.
  384. bs_buf.append(c)
  385. elif c == '"':
  386. # Double backslashes.
  387. result.append('\\' * len(bs_buf)*2)
  388. bs_buf = []
  389. result.append('\\"')
  390. else:
  391. # Normal char
  392. if bs_buf:
  393. result.extend(bs_buf)
  394. bs_buf = []
  395. result.append(c)
  396. # Add remaining backslashes, if any.
  397. if bs_buf:
  398. result.extend(bs_buf)
  399. if needquote:
  400. result.extend(bs_buf)
  401. result.append('"')
  402. return ''.join(result)
  403. ############################
  404. # End shell escaping code
  405. ############################
  406. if __name__ == '__main__':
  407. main()
  408. """
  409. ecoutils protocol version history
  410. ---------------------------------
  411. The version is ECO_VERSION module-level constant, and _eco_version key
  412. in the dictionary returned from ecoutils.get_profile().
  413. 1.0.1 - (boltons version 16.3.2+) Remove uuid dependency and add HAVE_URANDOM
  414. 1.0.0 - (boltons version 16.3.0-16.3.1) Initial release
  415. """