connection.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973
  1. #
  2. # A higher level module for using sockets (or Windows named pipes)
  3. #
  4. # multiprocessing/connection.py
  5. #
  6. # Copyright (c) 2006-2008, R Oudkerk
  7. # Licensed to PSF under a Contributor Agreement.
  8. #
  9. __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
  10. import io
  11. import os
  12. import sys
  13. import socket
  14. import struct
  15. import time
  16. import tempfile
  17. import itertools
  18. import _multiprocessing
  19. from . import util
  20. from . import AuthenticationError, BufferTooShort
  21. from .context import reduction
  22. _ForkingPickler = reduction.ForkingPickler
  23. try:
  24. import _winapi
  25. from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE
  26. except ImportError:
  27. if sys.platform == 'win32':
  28. raise
  29. _winapi = None
  30. #
  31. #
  32. #
  33. BUFSIZE = 8192
  34. # A very generous timeout when it comes to local connections...
  35. CONNECTION_TIMEOUT = 20.
  36. _mmap_counter = itertools.count()
  37. default_family = 'AF_INET'
  38. families = ['AF_INET']
  39. if hasattr(socket, 'AF_UNIX'):
  40. default_family = 'AF_UNIX'
  41. families += ['AF_UNIX']
  42. if sys.platform == 'win32':
  43. default_family = 'AF_PIPE'
  44. families += ['AF_PIPE']
  45. def _init_timeout(timeout=CONNECTION_TIMEOUT):
  46. return time.monotonic() + timeout
  47. def _check_timeout(t):
  48. return time.monotonic() > t
  49. #
  50. #
  51. #
  52. def arbitrary_address(family):
  53. '''
  54. Return an arbitrary free address for the given family
  55. '''
  56. if family == 'AF_INET':
  57. return ('localhost', 0)
  58. elif family == 'AF_UNIX':
  59. return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir())
  60. elif family == 'AF_PIPE':
  61. return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' %
  62. (os.getpid(), next(_mmap_counter)), dir="")
  63. else:
  64. raise ValueError('unrecognized family')
  65. def _validate_family(family):
  66. '''
  67. Checks if the family is valid for the current environment.
  68. '''
  69. if sys.platform != 'win32' and family == 'AF_PIPE':
  70. raise ValueError('Family %s is not recognized.' % family)
  71. if sys.platform == 'win32' and family == 'AF_UNIX':
  72. # double check
  73. if not hasattr(socket, family):
  74. raise ValueError('Family %s is not recognized.' % family)
  75. def address_type(address):
  76. '''
  77. Return the types of the address
  78. This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
  79. '''
  80. if type(address) == tuple:
  81. return 'AF_INET'
  82. elif type(address) is str and address.startswith('\\\\'):
  83. return 'AF_PIPE'
  84. elif type(address) is str or util.is_abstract_socket_namespace(address):
  85. return 'AF_UNIX'
  86. else:
  87. raise ValueError('address type of %r unrecognized' % address)
  88. #
  89. # Connection classes
  90. #
  91. class _ConnectionBase:
  92. _handle = None
  93. def __init__(self, handle, readable=True, writable=True):
  94. handle = handle.__index__()
  95. if handle < 0:
  96. raise ValueError("invalid handle")
  97. if not readable and not writable:
  98. raise ValueError(
  99. "at least one of `readable` and `writable` must be True")
  100. self._handle = handle
  101. self._readable = readable
  102. self._writable = writable
  103. # XXX should we use util.Finalize instead of a __del__?
  104. def __del__(self):
  105. if self._handle is not None:
  106. self._close()
  107. def _check_closed(self):
  108. if self._handle is None:
  109. raise OSError("handle is closed")
  110. def _check_readable(self):
  111. if not self._readable:
  112. raise OSError("connection is write-only")
  113. def _check_writable(self):
  114. if not self._writable:
  115. raise OSError("connection is read-only")
  116. def _bad_message_length(self):
  117. if self._writable:
  118. self._readable = False
  119. else:
  120. self.close()
  121. raise OSError("bad message length")
  122. @property
  123. def closed(self):
  124. """True if the connection is closed"""
  125. return self._handle is None
  126. @property
  127. def readable(self):
  128. """True if the connection is readable"""
  129. return self._readable
  130. @property
  131. def writable(self):
  132. """True if the connection is writable"""
  133. return self._writable
  134. def fileno(self):
  135. """File descriptor or handle of the connection"""
  136. self._check_closed()
  137. return self._handle
  138. def close(self):
  139. """Close the connection"""
  140. if self._handle is not None:
  141. try:
  142. self._close()
  143. finally:
  144. self._handle = None
  145. def send_bytes(self, buf, offset=0, size=None):
  146. """Send the bytes data from a bytes-like object"""
  147. self._check_closed()
  148. self._check_writable()
  149. m = memoryview(buf)
  150. # HACK for byte-indexing of non-bytewise buffers (e.g. array.array)
  151. if m.itemsize > 1:
  152. m = memoryview(bytes(m))
  153. n = len(m)
  154. if offset < 0:
  155. raise ValueError("offset is negative")
  156. if n < offset:
  157. raise ValueError("buffer length < offset")
  158. if size is None:
  159. size = n - offset
  160. elif size < 0:
  161. raise ValueError("size is negative")
  162. elif offset + size > n:
  163. raise ValueError("buffer length < offset + size")
  164. self._send_bytes(m[offset:offset + size])
  165. def send(self, obj):
  166. """Send a (picklable) object"""
  167. self._check_closed()
  168. self._check_writable()
  169. self._send_bytes(_ForkingPickler.dumps(obj))
  170. def recv_bytes(self, maxlength=None):
  171. """
  172. Receive bytes data as a bytes object.
  173. """
  174. self._check_closed()
  175. self._check_readable()
  176. if maxlength is not None and maxlength < 0:
  177. raise ValueError("negative maxlength")
  178. buf = self._recv_bytes(maxlength)
  179. if buf is None:
  180. self._bad_message_length()
  181. return buf.getvalue()
  182. def recv_bytes_into(self, buf, offset=0):
  183. """
  184. Receive bytes data into a writeable bytes-like object.
  185. Return the number of bytes read.
  186. """
  187. self._check_closed()
  188. self._check_readable()
  189. with memoryview(buf) as m:
  190. # Get bytesize of arbitrary buffer
  191. itemsize = m.itemsize
  192. bytesize = itemsize * len(m)
  193. if offset < 0:
  194. raise ValueError("negative offset")
  195. elif offset > bytesize:
  196. raise ValueError("offset too large")
  197. result = self._recv_bytes()
  198. size = result.tell()
  199. if bytesize < offset + size:
  200. raise BufferTooShort(result.getvalue())
  201. # Message can fit in dest
  202. result.seek(0)
  203. result.readinto(m[offset // itemsize :
  204. (offset + size) // itemsize])
  205. return size
  206. def recv(self):
  207. """Receive a (picklable) object"""
  208. self._check_closed()
  209. self._check_readable()
  210. buf = self._recv_bytes()
  211. return _ForkingPickler.loads(buf.getbuffer())
  212. def poll(self, timeout=0.0):
  213. """Whether there is any input available to be read"""
  214. self._check_closed()
  215. self._check_readable()
  216. return self._poll(timeout)
  217. def __enter__(self):
  218. return self
  219. def __exit__(self, exc_type, exc_value, exc_tb):
  220. self.close()
  221. if _winapi:
  222. class PipeConnection(_ConnectionBase):
  223. """
  224. Connection class based on a Windows named pipe.
  225. Overlapped I/O is used, so the handles must have been created
  226. with FILE_FLAG_OVERLAPPED.
  227. """
  228. _got_empty_message = False
  229. def _close(self, _CloseHandle=_winapi.CloseHandle):
  230. _CloseHandle(self._handle)
  231. def _send_bytes(self, buf):
  232. ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
  233. try:
  234. if err == _winapi.ERROR_IO_PENDING:
  235. waitres = _winapi.WaitForMultipleObjects(
  236. [ov.event], False, INFINITE)
  237. assert waitres == WAIT_OBJECT_0
  238. except:
  239. ov.cancel()
  240. raise
  241. finally:
  242. nwritten, err = ov.GetOverlappedResult(True)
  243. assert err == 0
  244. assert nwritten == len(buf)
  245. def _recv_bytes(self, maxsize=None):
  246. if self._got_empty_message:
  247. self._got_empty_message = False
  248. return io.BytesIO()
  249. else:
  250. bsize = 128 if maxsize is None else min(maxsize, 128)
  251. try:
  252. ov, err = _winapi.ReadFile(self._handle, bsize,
  253. overlapped=True)
  254. try:
  255. if err == _winapi.ERROR_IO_PENDING:
  256. waitres = _winapi.WaitForMultipleObjects(
  257. [ov.event], False, INFINITE)
  258. assert waitres == WAIT_OBJECT_0
  259. except:
  260. ov.cancel()
  261. raise
  262. finally:
  263. nread, err = ov.GetOverlappedResult(True)
  264. if err == 0:
  265. f = io.BytesIO()
  266. f.write(ov.getbuffer())
  267. return f
  268. elif err == _winapi.ERROR_MORE_DATA:
  269. return self._get_more_data(ov, maxsize)
  270. except OSError as e:
  271. if e.winerror == _winapi.ERROR_BROKEN_PIPE:
  272. raise EOFError
  273. else:
  274. raise
  275. raise RuntimeError("shouldn't get here; expected KeyboardInterrupt")
  276. def _poll(self, timeout):
  277. if (self._got_empty_message or
  278. _winapi.PeekNamedPipe(self._handle)[0] != 0):
  279. return True
  280. return bool(wait([self], timeout))
  281. def _get_more_data(self, ov, maxsize):
  282. buf = ov.getbuffer()
  283. f = io.BytesIO()
  284. f.write(buf)
  285. left = _winapi.PeekNamedPipe(self._handle)[1]
  286. assert left > 0
  287. if maxsize is not None and len(buf) + left > maxsize:
  288. self._bad_message_length()
  289. ov, err = _winapi.ReadFile(self._handle, left, overlapped=True)
  290. rbytes, err = ov.GetOverlappedResult(True)
  291. assert err == 0
  292. assert rbytes == left
  293. f.write(ov.getbuffer())
  294. return f
  295. class Connection(_ConnectionBase):
  296. """
  297. Connection class based on an arbitrary file descriptor (Unix only), or
  298. a socket handle (Windows).
  299. """
  300. if _winapi:
  301. def _close(self, _close=_multiprocessing.closesocket):
  302. _close(self._handle)
  303. _write = _multiprocessing.send
  304. _read = _multiprocessing.recv
  305. else:
  306. def _close(self, _close=os.close):
  307. _close(self._handle)
  308. _write = os.write
  309. _read = os.read
  310. def _send(self, buf, write=_write):
  311. remaining = len(buf)
  312. while True:
  313. n = write(self._handle, buf)
  314. remaining -= n
  315. if remaining == 0:
  316. break
  317. buf = buf[n:]
  318. def _recv(self, size, read=_read):
  319. buf = io.BytesIO()
  320. handle = self._handle
  321. remaining = size
  322. while remaining > 0:
  323. chunk = read(handle, remaining)
  324. n = len(chunk)
  325. if n == 0:
  326. if remaining == size:
  327. raise EOFError
  328. else:
  329. raise OSError("got end of file during message")
  330. buf.write(chunk)
  331. remaining -= n
  332. return buf
  333. def _send_bytes(self, buf):
  334. n = len(buf)
  335. if n > 0x7fffffff:
  336. pre_header = struct.pack("!i", -1)
  337. header = struct.pack("!Q", n)
  338. self._send(pre_header)
  339. self._send(header)
  340. self._send(buf)
  341. else:
  342. # For wire compatibility with 3.7 and lower
  343. header = struct.pack("!i", n)
  344. if n > 16384:
  345. # The payload is large so Nagle's algorithm won't be triggered
  346. # and we'd better avoid the cost of concatenation.
  347. self._send(header)
  348. self._send(buf)
  349. else:
  350. # Issue #20540: concatenate before sending, to avoid delays due
  351. # to Nagle's algorithm on a TCP socket.
  352. # Also note we want to avoid sending a 0-length buffer separately,
  353. # to avoid "broken pipe" errors if the other end closed the pipe.
  354. self._send(header + buf)
  355. def _recv_bytes(self, maxsize=None):
  356. buf = self._recv(4)
  357. size, = struct.unpack("!i", buf.getvalue())
  358. if size == -1:
  359. buf = self._recv(8)
  360. size, = struct.unpack("!Q", buf.getvalue())
  361. if maxsize is not None and size > maxsize:
  362. return None
  363. return self._recv(size)
  364. def _poll(self, timeout):
  365. r = wait([self], timeout)
  366. return bool(r)
  367. #
  368. # Public functions
  369. #
  370. class Listener(object):
  371. '''
  372. Returns a listener object.
  373. This is a wrapper for a bound socket which is 'listening' for
  374. connections, or for a Windows named pipe.
  375. '''
  376. def __init__(self, address=None, family=None, backlog=1, authkey=None):
  377. family = family or (address and address_type(address)) \
  378. or default_family
  379. address = address or arbitrary_address(family)
  380. _validate_family(family)
  381. if family == 'AF_PIPE':
  382. self._listener = PipeListener(address, backlog)
  383. else:
  384. self._listener = SocketListener(address, family, backlog)
  385. if authkey is not None and not isinstance(authkey, bytes):
  386. raise TypeError('authkey should be a byte string')
  387. self._authkey = authkey
  388. def accept(self):
  389. '''
  390. Accept a connection on the bound socket or named pipe of `self`.
  391. Returns a `Connection` object.
  392. '''
  393. if self._listener is None:
  394. raise OSError('listener is closed')
  395. c = self._listener.accept()
  396. if self._authkey:
  397. deliver_challenge(c, self._authkey)
  398. answer_challenge(c, self._authkey)
  399. return c
  400. def close(self):
  401. '''
  402. Close the bound socket or named pipe of `self`.
  403. '''
  404. listener = self._listener
  405. if listener is not None:
  406. self._listener = None
  407. listener.close()
  408. @property
  409. def address(self):
  410. return self._listener._address
  411. @property
  412. def last_accepted(self):
  413. return self._listener._last_accepted
  414. def __enter__(self):
  415. return self
  416. def __exit__(self, exc_type, exc_value, exc_tb):
  417. self.close()
  418. def Client(address, family=None, authkey=None):
  419. '''
  420. Returns a connection to the address of a `Listener`
  421. '''
  422. family = family or address_type(address)
  423. _validate_family(family)
  424. if family == 'AF_PIPE':
  425. c = PipeClient(address)
  426. else:
  427. c = SocketClient(address)
  428. if authkey is not None and not isinstance(authkey, bytes):
  429. raise TypeError('authkey should be a byte string')
  430. if authkey is not None:
  431. answer_challenge(c, authkey)
  432. deliver_challenge(c, authkey)
  433. return c
  434. if sys.platform != 'win32':
  435. def Pipe(duplex=True):
  436. '''
  437. Returns pair of connection objects at either end of a pipe
  438. '''
  439. if duplex:
  440. s1, s2 = socket.socketpair()
  441. s1.setblocking(True)
  442. s2.setblocking(True)
  443. c1 = Connection(s1.detach())
  444. c2 = Connection(s2.detach())
  445. else:
  446. fd1, fd2 = os.pipe()
  447. c1 = Connection(fd1, writable=False)
  448. c2 = Connection(fd2, readable=False)
  449. return c1, c2
  450. else:
  451. def Pipe(duplex=True):
  452. '''
  453. Returns pair of connection objects at either end of a pipe
  454. '''
  455. address = arbitrary_address('AF_PIPE')
  456. if duplex:
  457. openmode = _winapi.PIPE_ACCESS_DUPLEX
  458. access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
  459. obsize, ibsize = BUFSIZE, BUFSIZE
  460. else:
  461. openmode = _winapi.PIPE_ACCESS_INBOUND
  462. access = _winapi.GENERIC_WRITE
  463. obsize, ibsize = 0, BUFSIZE
  464. h1 = _winapi.CreateNamedPipe(
  465. address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
  466. _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
  467. _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
  468. _winapi.PIPE_WAIT,
  469. 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER,
  470. # default security descriptor: the handle cannot be inherited
  471. _winapi.NULL
  472. )
  473. h2 = _winapi.CreateFile(
  474. address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
  475. _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
  476. )
  477. _winapi.SetNamedPipeHandleState(
  478. h2, _winapi.PIPE_READMODE_MESSAGE, None, None
  479. )
  480. overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True)
  481. _, err = overlapped.GetOverlappedResult(True)
  482. assert err == 0
  483. c1 = PipeConnection(h1, writable=duplex)
  484. c2 = PipeConnection(h2, readable=duplex)
  485. return c1, c2
  486. #
  487. # Definitions for connections based on sockets
  488. #
  489. class SocketListener(object):
  490. '''
  491. Representation of a socket which is bound to an address and listening
  492. '''
  493. def __init__(self, address, family, backlog=1):
  494. self._socket = socket.socket(getattr(socket, family))
  495. try:
  496. # SO_REUSEADDR has different semantics on Windows (issue #2550).
  497. if os.name == 'posix':
  498. self._socket.setsockopt(socket.SOL_SOCKET,
  499. socket.SO_REUSEADDR, 1)
  500. self._socket.setblocking(True)
  501. self._socket.bind(address)
  502. self._socket.listen(backlog)
  503. self._address = self._socket.getsockname()
  504. except OSError:
  505. self._socket.close()
  506. raise
  507. self._family = family
  508. self._last_accepted = None
  509. if family == 'AF_UNIX' and not util.is_abstract_socket_namespace(address):
  510. # Linux abstract socket namespaces do not need to be explicitly unlinked
  511. self._unlink = util.Finalize(
  512. self, os.unlink, args=(address,), exitpriority=0
  513. )
  514. else:
  515. self._unlink = None
  516. def accept(self):
  517. s, self._last_accepted = self._socket.accept()
  518. s.setblocking(True)
  519. return Connection(s.detach())
  520. def close(self):
  521. try:
  522. self._socket.close()
  523. finally:
  524. unlink = self._unlink
  525. if unlink is not None:
  526. self._unlink = None
  527. unlink()
  528. def SocketClient(address):
  529. '''
  530. Return a connection object connected to the socket given by `address`
  531. '''
  532. family = address_type(address)
  533. with socket.socket( getattr(socket, family) ) as s:
  534. s.setblocking(True)
  535. s.connect(address)
  536. return Connection(s.detach())
  537. #
  538. # Definitions for connections based on named pipes
  539. #
  540. if sys.platform == 'win32':
  541. class PipeListener(object):
  542. '''
  543. Representation of a named pipe
  544. '''
  545. def __init__(self, address, backlog=None):
  546. self._address = address
  547. self._handle_queue = [self._new_handle(first=True)]
  548. self._last_accepted = None
  549. util.sub_debug('listener created with address=%r', self._address)
  550. self.close = util.Finalize(
  551. self, PipeListener._finalize_pipe_listener,
  552. args=(self._handle_queue, self._address), exitpriority=0
  553. )
  554. def _new_handle(self, first=False):
  555. flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED
  556. if first:
  557. flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
  558. return _winapi.CreateNamedPipe(
  559. self._address, flags,
  560. _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
  561. _winapi.PIPE_WAIT,
  562. _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
  563. _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
  564. )
  565. def accept(self):
  566. self._handle_queue.append(self._new_handle())
  567. handle = self._handle_queue.pop(0)
  568. try:
  569. ov = _winapi.ConnectNamedPipe(handle, overlapped=True)
  570. except OSError as e:
  571. if e.winerror != _winapi.ERROR_NO_DATA:
  572. raise
  573. # ERROR_NO_DATA can occur if a client has already connected,
  574. # written data and then disconnected -- see Issue 14725.
  575. else:
  576. try:
  577. res = _winapi.WaitForMultipleObjects(
  578. [ov.event], False, INFINITE)
  579. except:
  580. ov.cancel()
  581. _winapi.CloseHandle(handle)
  582. raise
  583. finally:
  584. _, err = ov.GetOverlappedResult(True)
  585. assert err == 0
  586. return PipeConnection(handle)
  587. @staticmethod
  588. def _finalize_pipe_listener(queue, address):
  589. util.sub_debug('closing listener with address=%r', address)
  590. for handle in queue:
  591. _winapi.CloseHandle(handle)
  592. def PipeClient(address):
  593. '''
  594. Return a connection object connected to the pipe given by `address`
  595. '''
  596. t = _init_timeout()
  597. while 1:
  598. try:
  599. _winapi.WaitNamedPipe(address, 1000)
  600. h = _winapi.CreateFile(
  601. address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE,
  602. 0, _winapi.NULL, _winapi.OPEN_EXISTING,
  603. _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
  604. )
  605. except OSError as e:
  606. if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT,
  607. _winapi.ERROR_PIPE_BUSY) or _check_timeout(t):
  608. raise
  609. else:
  610. break
  611. else:
  612. raise
  613. _winapi.SetNamedPipeHandleState(
  614. h, _winapi.PIPE_READMODE_MESSAGE, None, None
  615. )
  616. return PipeConnection(h)
  617. #
  618. # Authentication stuff
  619. #
  620. MESSAGE_LENGTH = 20
  621. CHALLENGE = b'#CHALLENGE#'
  622. WELCOME = b'#WELCOME#'
  623. FAILURE = b'#FAILURE#'
  624. def deliver_challenge(connection, authkey):
  625. import hmac
  626. if not isinstance(authkey, bytes):
  627. raise ValueError(
  628. "Authkey must be bytes, not {0!s}".format(type(authkey)))
  629. message = os.urandom(MESSAGE_LENGTH)
  630. connection.send_bytes(CHALLENGE + message)
  631. digest = hmac.new(authkey, message, 'md5').digest()
  632. response = connection.recv_bytes(256) # reject large message
  633. if response == digest:
  634. connection.send_bytes(WELCOME)
  635. else:
  636. connection.send_bytes(FAILURE)
  637. raise AuthenticationError('digest received was wrong')
  638. def answer_challenge(connection, authkey):
  639. import hmac
  640. if not isinstance(authkey, bytes):
  641. raise ValueError(
  642. "Authkey must be bytes, not {0!s}".format(type(authkey)))
  643. message = connection.recv_bytes(256) # reject large message
  644. assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
  645. message = message[len(CHALLENGE):]
  646. digest = hmac.new(authkey, message, 'md5').digest()
  647. connection.send_bytes(digest)
  648. response = connection.recv_bytes(256) # reject large message
  649. if response != WELCOME:
  650. raise AuthenticationError('digest sent was rejected')
  651. #
  652. # Support for using xmlrpclib for serialization
  653. #
  654. class ConnectionWrapper(object):
  655. def __init__(self, conn, dumps, loads):
  656. self._conn = conn
  657. self._dumps = dumps
  658. self._loads = loads
  659. for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
  660. obj = getattr(conn, attr)
  661. setattr(self, attr, obj)
  662. def send(self, obj):
  663. s = self._dumps(obj)
  664. self._conn.send_bytes(s)
  665. def recv(self):
  666. s = self._conn.recv_bytes()
  667. return self._loads(s)
  668. def _xml_dumps(obj):
  669. return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8')
  670. def _xml_loads(s):
  671. (obj,), method = xmlrpclib.loads(s.decode('utf-8'))
  672. return obj
  673. class XmlListener(Listener):
  674. def accept(self):
  675. global xmlrpclib
  676. import xmlrpc.client as xmlrpclib
  677. obj = Listener.accept(self)
  678. return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
  679. def XmlClient(*args, **kwds):
  680. global xmlrpclib
  681. import xmlrpc.client as xmlrpclib
  682. return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)
  683. #
  684. # Wait
  685. #
  686. if sys.platform == 'win32':
  687. def _exhaustive_wait(handles, timeout):
  688. # Return ALL handles which are currently signalled. (Only
  689. # returning the first signalled might create starvation issues.)
  690. L = list(handles)
  691. ready = []
  692. while L:
  693. res = _winapi.WaitForMultipleObjects(L, False, timeout)
  694. if res == WAIT_TIMEOUT:
  695. break
  696. elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):
  697. res -= WAIT_OBJECT_0
  698. elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L):
  699. res -= WAIT_ABANDONED_0
  700. else:
  701. raise RuntimeError('Should not get here')
  702. ready.append(L[res])
  703. L = L[res+1:]
  704. timeout = 0
  705. return ready
  706. _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED}
  707. def wait(object_list, timeout=None):
  708. '''
  709. Wait till an object in object_list is ready/readable.
  710. Returns list of those objects in object_list which are ready/readable.
  711. '''
  712. if timeout is None:
  713. timeout = INFINITE
  714. elif timeout < 0:
  715. timeout = 0
  716. else:
  717. timeout = int(timeout * 1000 + 0.5)
  718. object_list = list(object_list)
  719. waithandle_to_obj = {}
  720. ov_list = []
  721. ready_objects = set()
  722. ready_handles = set()
  723. try:
  724. for o in object_list:
  725. try:
  726. fileno = getattr(o, 'fileno')
  727. except AttributeError:
  728. waithandle_to_obj[o.__index__()] = o
  729. else:
  730. # start an overlapped read of length zero
  731. try:
  732. ov, err = _winapi.ReadFile(fileno(), 0, True)
  733. except OSError as e:
  734. ov, err = None, e.winerror
  735. if err not in _ready_errors:
  736. raise
  737. if err == _winapi.ERROR_IO_PENDING:
  738. ov_list.append(ov)
  739. waithandle_to_obj[ov.event] = o
  740. else:
  741. # If o.fileno() is an overlapped pipe handle and
  742. # err == 0 then there is a zero length message
  743. # in the pipe, but it HAS NOT been consumed...
  744. if ov and sys.getwindowsversion()[:2] >= (6, 2):
  745. # ... except on Windows 8 and later, where
  746. # the message HAS been consumed.
  747. try:
  748. _, err = ov.GetOverlappedResult(False)
  749. except OSError as e:
  750. err = e.winerror
  751. if not err and hasattr(o, '_got_empty_message'):
  752. o._got_empty_message = True
  753. ready_objects.add(o)
  754. timeout = 0
  755. ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout)
  756. finally:
  757. # request that overlapped reads stop
  758. for ov in ov_list:
  759. ov.cancel()
  760. # wait for all overlapped reads to stop
  761. for ov in ov_list:
  762. try:
  763. _, err = ov.GetOverlappedResult(True)
  764. except OSError as e:
  765. err = e.winerror
  766. if err not in _ready_errors:
  767. raise
  768. if err != _winapi.ERROR_OPERATION_ABORTED:
  769. o = waithandle_to_obj[ov.event]
  770. ready_objects.add(o)
  771. if err == 0:
  772. # If o.fileno() is an overlapped pipe handle then
  773. # a zero length message HAS been consumed.
  774. if hasattr(o, '_got_empty_message'):
  775. o._got_empty_message = True
  776. ready_objects.update(waithandle_to_obj[h] for h in ready_handles)
  777. return [o for o in object_list if o in ready_objects]
  778. else:
  779. import selectors
  780. # poll/select have the advantage of not requiring any extra file
  781. # descriptor, contrarily to epoll/kqueue (also, they require a single
  782. # syscall).
  783. if hasattr(selectors, 'PollSelector'):
  784. _WaitSelector = selectors.PollSelector
  785. else:
  786. _WaitSelector = selectors.SelectSelector
  787. def wait(object_list, timeout=None):
  788. '''
  789. Wait till an object in object_list is ready/readable.
  790. Returns list of those objects in object_list which are ready/readable.
  791. '''
  792. with _WaitSelector() as selector:
  793. for obj in object_list:
  794. selector.register(obj, selectors.EVENT_READ)
  795. if timeout is not None:
  796. deadline = time.monotonic() + timeout
  797. while True:
  798. ready = selector.select(timeout)
  799. if ready:
  800. return [key.fileobj for (key, events) in ready]
  801. else:
  802. if timeout is not None:
  803. timeout = deadline - time.monotonic()
  804. if timeout < 0:
  805. return ready
  806. #
  807. # Make connection and socket objects sharable if possible
  808. #
  809. if sys.platform == 'win32':
  810. def reduce_connection(conn):
  811. handle = conn.fileno()
  812. with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s:
  813. from . import resource_sharer
  814. ds = resource_sharer.DupSocket(s)
  815. return rebuild_connection, (ds, conn.readable, conn.writable)
  816. def rebuild_connection(ds, readable, writable):
  817. sock = ds.detach()
  818. return Connection(sock.detach(), readable, writable)
  819. reduction.register(Connection, reduce_connection)
  820. def reduce_pipe_connection(conn):
  821. access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) |
  822. (_winapi.FILE_GENERIC_WRITE if conn.writable else 0))
  823. dh = reduction.DupHandle(conn.fileno(), access)
  824. return rebuild_pipe_connection, (dh, conn.readable, conn.writable)
  825. def rebuild_pipe_connection(dh, readable, writable):
  826. handle = dh.detach()
  827. return PipeConnection(handle, readable, writable)
  828. reduction.register(PipeConnection, reduce_pipe_connection)
  829. else:
  830. def reduce_connection(conn):
  831. df = reduction.DupFd(conn.fileno())
  832. return rebuild_connection, (df, conn.readable, conn.writable)
  833. def rebuild_connection(df, readable, writable):
  834. fd = df.detach()
  835. return Connection(fd, readable, writable)
  836. reduction.register(Connection, reduce_connection)