Gnumed.pycommon.gmBackendListener

Package Gnumed :: Package pycommon :: Module gmBackendListener |
[frames] | no frames]

# Source Code for Module Gnumed.pycommon.gmBackendListener

      1  __doc__ = """GNUmed database backend listener.
      2
      3  This module implements threaded listening for asynchronuous
      4  notifications from the database backend.
      5  """
      6  #=====================================================================
      7  __author__ = "H. Herb <hherb@gnumed.net>, K.Hilbert <karsten.hilbert@gmx.net>"
      8  __license__ = "GPL v2 or later"
      9
     10  import sys
     11  import time
     12  import threading
     13  import select
     14  import logging
     15
     16
     17  if __name__ == '__main__':
     18          sys.path.insert(0, '../../')
     19  from Gnumed.pycommon import gmDispatcher
     20  from Gnumed.pycommon import gmBorg
     21
     22
     23  _log = logging.getLogger('gm.db')
     24
     25
     26  signals2listen4 = [
     27          'db_maintenance_warning',               # warns of impending maintenance and asks for disconnect
     28          'db_maintenance_disconnect',    # announces a forced disconnect and disconnects
     29          'gm_table_mod'                                  # sent for any (registered) table modification, payload contains details
     30  ]
     31
     32  #=====================================================================
     33 -class gmBackendListener(gmBorg.cBorg):
     34
     35 -        def __init__(self, conn=None, poll_interval=3):
     36
     37                  try:
     38                          self.already_inited
     39                          return
     40
     41                  except AttributeError:
     42                          pass
     43
     44                  self.debug = False
     45                  self.__notifications_received = 0
     46                  self.__messages_sent = 0
     47
     48                  _log.info('starting backend notifications listener thread')
     49
     50                  # the listener thread will regularly try to acquire
     51                  # this lock, when it succeeds it will quit
     52                  self._quit_lock = threading.Lock()
     53                  # take the lock now so it cannot be taken by the worker
     54                  # thread until it is released in shutdown()
     55                  if not self._quit_lock.acquire(0):
     56                          _log.error('cannot acquire thread-quit lock, aborting')
     57                          raise EnvironmentError("cannot acquire thread-quit lock")
     58
     59                  self._conn = conn
     60                  self.backend_pid = self._conn.get_backend_pid()
     61                  _log.debug('notification listener connection has backend PID [%s]', self.backend_pid)
     62                  self._conn.set_isolation_level(0)               # autocommit mode = psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT
     63                  self._cursor = self._conn.cursor()
     64                  try:
     65                          self._conn_fd = self._conn.fileno()
     66                  except AttributeError:
     67                          self._conn_fd = self._cursor.fileno()
     68                  self._conn_lock = threading.Lock()              # lock for access to connection object
     69
     70                  self.__register_interests()
     71
     72                  # check for messages every 'poll_interval' seconds
     73                  self._poll_interval = poll_interval
     74                  self._listener_thread = None
     75                  self.__start_thread()
     76
     77                  self.already_inited = True
     78
     79          #-------------------------------
     80          # public API
     81          #-------------------------------
     82 -        def shutdown(self):
     83                  _log.debug('received %s notifications', self.__notifications_received)
     84                  _log.debug('sent %s messages', self.__messages_sent)
     85
     86                  if self._listener_thread is None:
     87                          self.__shutdown_connection()
     88                          return
     89
     90                  _log.info('stopping backend notifications listener thread')
     91                  self._quit_lock.release()
     92                  try:
     93                          # give the worker thread time to terminate
     94                          self._listener_thread.join(self._poll_interval+2.0)
     95                          try:
     96                                  if self._listener_thread.isAlive():
     97                                          _log.error('listener thread still alive after join()')
     98                                          _log.debug('active threads: %s' % threading.enumerate())
     99                          except Exception:
    100                                  pass
    101                  except Exception:
    102                          print(sys.exc_info())
    103
    104                  self._listener_thread = None
    105
    106                  try:
    107                          self.__unregister_unspecific_notifications()
    108                  except Exception:
    109                          _log.exception('unable to unregister unspecific notifications')
    110
    111                  self.__shutdown_connection()
    112
    113                  return
    114          #-------------------------------
    115          # event handlers
    116          #-------------------------------
    117          # internal helpers
    118          #-------------------------------
    119 -        def __register_interests(self):
    120                  # determine unspecific notifications
    121                  self.unspecific_notifications = signals2listen4
    122                  _log.info('configured unspecific notifications:')
    123                  _log.info('%s' % self.unspecific_notifications)
    124                  gmDispatcher.known_signals.extend(self.unspecific_notifications)
    125
    126                  # listen to unspecific notifications
    127                  self.__register_unspecific_notifications()
    128
    129          #-------------------------------
    130 -        def __register_unspecific_notifications(self):
    131                  for sig in self.unspecific_notifications:
    132                          _log.info('starting to listen for [%s]' % sig)
    133                          cmd = 'LISTEN "%s"' % sig
    134                          self._conn_lock.acquire(1)
    135                          try:
    136                                  self._cursor.execute(cmd)
    137                          finally:
    138                                  self._conn_lock.release()
    139
    140          #-------------------------------
    141 -        def __unregister_unspecific_notifications(self):
    142                  for sig in self.unspecific_notifications:
    143                          _log.info('stopping to listen for [%s]' % sig)
    144                          cmd = 'UNLISTEN "%s"' % sig
    145                          self._conn_lock.acquire(1)
    146                          try:
    147                                  self._cursor.execute(cmd)
    148                          finally:
    149                                  self._conn_lock.release()
    150
    151          #-------------------------------
    152 -        def __shutdown_connection(self):
    153                  _log.debug('shutting down connection with backend PID [%s]', self.backend_pid)
    154                  self._conn_lock.acquire(1)
    155                  try:
    156                          self._conn.rollback()
    157                  except Exception:
    158                          pass
    159                  finally:
    160                          self._conn_lock.release()
    161
    162          #-------------------------------
    163 -        def __start_thread(self):
    164                  if self._conn is None:
    165                          raise ValueError("no connection to backend available, useless to start thread")
    166
    167                  self._listener_thread = threading.Thread (
    168                          target = self._process_notifications,
    169                          name = self.__class__.__name__,
    170                          daemon = True
    171                  )
    172                  _log.info('starting listener thread')
    173                  self._listener_thread.start()
    174
    175          #-------------------------------
    176          # the actual thread code
    177          #-------------------------------
    178 -        def _process_notifications(self):
    179
    180                  # loop until quitting
    181                  _have_quit_lock = None
    182                  while not _have_quit_lock:
    183                          # quitting ?
    184                          if self._quit_lock.acquire(0):
    185                                  break
    186
    187                          # wait at most self._poll_interval for new data
    188                          self._conn_lock.acquire(1)
    189                          try:
    190                                  ready_input_sockets = select.select([self._conn_fd], [], [], self._poll_interval)[0]
    191                          finally:
    192                                  self._conn_lock.release()
    193                          # any input available ?
    194                          if len(ready_input_sockets) == 0:
    195                                  # no, select.select() timed out
    196                                  # give others a chance to grab the conn lock (eg listen/unlisten)
    197                                  time.sleep(0.3)
    198                                  continue
    199                          # data available, wait for it to fully arrive
    200                          self._conn_lock.acquire(1)
    201                          try:
    202                                  self._conn.poll()
    203                          finally:
    204                                  self._conn_lock.release()
    205                          # any notifications ?
    206                          while len(self._conn.notifies) > 0:
    207                                  # if self._quit_lock can be acquired we may be in
    208                                  # __del__ in which case gmDispatcher is not
    209                                  # guaranteed to exist anymore
    210                                  if self._quit_lock.acquire(0):
    211                                          _have_quit_lock = 1
    212                                          break
    213
    214                                  self._conn_lock.acquire(1)
    215                                  try:
    216                                          notification = self._conn.notifies.pop()
    217                                  finally:
    218                                          self._conn_lock.release()
    219                                  self.__notifications_received += 1
    220                                  if self.debug:
    221                                          print(notification)
    222                                  _log.debug('#%s: %s (first param is PID of sending backend)', self.__notifications_received, notification)
    223                                  # decode payload
    224                                  payload = notification.payload.split('::')
    225                                  operation = None
    226                                  table = None
    227                                  pk_column_name = None
    228                                  pk_of_row = None
    229                                  pk_identity = None
    230                                  for item in payload:
    231                                          if item.startswith('operation='):
    232                                                  operation = item.split('=')[1]
    233                                          if item.startswith('table='):
    234                                                  table = item.split('=')[1]
    235                                          if item.startswith('PK name='):
    236                                                  pk_column_name = item.split('=')[1]
    237                                          if item.startswith('row PK='):
    238                                                  pk_of_row = int(item.split('=')[1])
    239                                          if item.startswith('person PK='):
    240                                                  try:
    241                                                          pk_identity = int(item.split('=')[1])
    242                                                  except ValueError:
    243                                                          _log.exception('error in change notification trigger')
    244                                                          pk_identity = -1
    245                                  # try sending intra-client signals:
    246                                  # 1) generic signal
    247                                  self.__messages_sent += 1
    248                                  try:
    249                                          results = gmDispatcher.send (
    250                                                  signal = notification.channel,
    251                                                  originated_in_database = True,
    252                                                  listener_pid = self.backend_pid,
    253                                                  sending_backend_pid = notification.pid,
    254                                                  pk_identity = pk_identity,
    255                                                  operation = operation,
    256                                                  table = table,
    257                                                  pk_column_name = pk_column_name,
    258                                                  pk_of_row = pk_of_row,
    259                                                  message_index = self.__messages_sent,
    260                                                  notification_index = self.__notifications_received
    261                                          )
    262                                  except Exception:
    263                                          print("problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (notification.channel, notification.pid))
    264                                          print(sys.exc_info())
    265                                  # 2) dynamically emulated old style table specific signals
    266                                  if table is not None:
    267                                          self.__messages_sent += 1
    268                                          signal = '%s_mod_db' % table
    269                                          _log.debug('emulating old-style table specific signal [%s]', signal)
    270                                          try:
    271                                                  results = gmDispatcher.send (
    272                                                          signal = signal,
    273                                                          originated_in_database = True,
    274                                                          listener_pid = self.backend_pid,
    275                                                          sending_backend_pid = notification.pid,
    276                                                          pk_identity = pk_identity,
    277                                                          operation = operation,
    278                                                          table = table,
    279                                                          pk_column_name = pk_column_name,
    280                                                          pk_of_row = pk_of_row,
    281                                                          message_index = self.__messages_sent,
    282                                                          notification_index = self.__notifications_received
    283                                                  )
    284                                          except Exception:
    285                                                  print("problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (signal, notification.pid))
    286                                                  print(sys.exc_info())
    287
    288                                  # there *may* be more pending notifications but
    289                                  # we don't care when quitting
    290                                  if self._quit_lock.acquire(0):
    291                                          _have_quit_lock = 1
    292                                          break
    293
    294                  # exit thread activity
    295                  return
    296
    297  #=====================================================================
    298  # main
    299  #=====================================================================
    300  if __name__ == "__main__":
    301
    302          if len(sys.argv) < 2:
    303                  sys.exit()
    304
    305          if sys.argv[1] not in ['test', 'monitor']:
    306                  sys.exit()
    307
    308
    309          notifies = 0
    310
    311          from Gnumed.pycommon import gmPG2, gmI18N
    312          from Gnumed.business import gmPerson, gmPersonSearch
    313
    314          gmI18N.activate_locale()
    315          gmI18N.install_domain(domain='gnumed')
    316          #-------------------------------
    317 -        def run_test():
    318
    319                  #-------------------------------
    320                  def dummy(n):
    321                          return float(n)*n/float(1+n)
    322                  #-------------------------------
    323                  def OnPatientModified():
    324                          global notifies
    325                          notifies += 1
    326                          sys.stdout.flush()
    327                          print("\nBackend says: patient data has been modified (%s. notification)" % notifies)
    328                  #-------------------------------
    329                  try:
    330                          n = int(sys.argv[2])
    331                  except Exception:
    332                          print("You can set the number of iterations\nwith the second command line argument")
    333                          n = 100000
    334
    335                  # try loop without backend listener
    336                  print("Looping", n, "times through dummy function")
    337                  i = 0
    338                  t1 = time.time()
    339                  while i < n:
    340                          r = dummy(i)
    341                          i += 1
    342                  t2 = time.time()
    343                  t_nothreads = t2-t1
    344                  print("Without backend thread, it took", t_nothreads, "seconds")
    345
    346                  listener = gmBackendListener(conn = gmPG2.get_raw_connection())
    347
    348                  # now try with listener to measure impact
    349                  print("Now in a new shell connect psql to the")
    350                  print("database <gnumed_v9> on localhost, return")
    351                  print("here and hit <enter> to continue.")
    352                  input('hit <enter> when done starting psql')
    353                  print("You now have about 30 seconds to go")
    354                  print("to the psql shell and type")
    355                  print(" notify patient_changed<enter>")
    356                  print("several times.")
    357                  print("This should trigger our backend listening callback.")
    358                  print("You can also try to stop the demo with Ctrl-C !")
    359
    360                  listener.register_callback('patient_changed', OnPatientModified)
    361
    362                  try:
    363                          counter = 0
    364                          while counter < 20:
    365                                  counter += 1
    366                                  time.sleep(1)
    367                                  sys.stdout.flush()
    368                                  print('.')
    369                          print("Looping",n,"times through dummy function")
    370                          i = 0
    371                          t1 = time.time()
    372                          while i < n:
    373                                  r = dummy(i)
    374                                  i += 1
    375                          t2 = time.time()
    376                          t_threaded = t2-t1
    377                          print("With backend thread, it took", t_threaded, "seconds")
    378                          print("Difference:", t_threaded-t_nothreads)
    379                  except KeyboardInterrupt:
    380                          print("cancelled by user")
    381
    382                  listener.shutdown()
    383                  listener.unregister_callback('patient_changed', OnPatientModified)
    384          #-------------------------------
    385 -        def run_monitor():
    386
    387                  print("starting up backend notifications monitor")
    388
    389                  def monitoring_callback(*args, **kwargs):
    390                          try:
    391                                  kwargs['originated_in_database']
    392                                  print('==> got notification from database "%s":' % kwargs['signal'])
    393                          except KeyError:
    394                                  print('==> received signal from client: "%s"' % kwargs['signal'])
    395                          del kwargs['signal']
    396                          for key in kwargs.keys():
    397                                  print('    [%s]: %s' % (key, kwargs[key]))
    398
    399                  gmDispatcher.connect(receiver = monitoring_callback)
    400
    401                  listener = gmBackendListener(conn = gmPG2.get_raw_connection())
    402                  print("listening for the following notifications:")
    403                  print("1) unspecific:")
    404                  for sig in listener.unspecific_notifications:
    405                          print('   - %s' % sig)
    406
    407                  while True:
    408                          pat = gmPersonSearch.ask_for_patient()
    409                          if pat is None:
    410                                  break
    411                          print("found patient", pat)
    412                          gmPerson.set_active_patient(patient=pat)
    413                          print("now waiting for notifications, hit <ENTER> to select another patient")
    414                          input()
    415
    416                  print("cleanup")
    417                  listener.shutdown()
    418
    419                  print("shutting down backend notifications monitor")
    420
    421          #-------------------------------
    422          if sys.argv[1] == 'monitor':
    423                  run_monitor()
    424          else:
    425                  run_test()
    426
    427  #=====================================================================
    428