Gnumed.pycommon.gmTools

Package Gnumed :: Package pycommon :: Module gmTools |
[frames] | no frames]

# Source Code for Module Gnumed.pycommon.gmTools

       1  # -*- coding: utf-8 -*-
       2
       3
       4  __doc__ = """GNUmed general tools."""
       5
       6  #===========================================================================
       7  __author__ = "K. Hilbert <Karsten.Hilbert@gmx.net>"
       8  __license__ = "GPL v2 or later (details at http://www.gnu.org)"
       9
      10  # std libs
      11  import sys
      12  import os
      13  import os.path
      14  import csv
      15  import tempfile
      16  import logging
      17  import hashlib
      18  import platform
      19  import subprocess
      20  import decimal
      21  import getpass
      22  import io
      23  import functools
      24  import json
      25  import shutil
      26  import zipfile
      27  import datetime as pydt
      28  import re as regex
      29  import xml.sax.saxutils as xml_tools
      30  # old:
      31  import pickle, zlib
      32  # docutils
      33  du_core = None
      34
      35
      36  # GNUmed libs
      37  if __name__ == '__main__':
      38          sys.path.insert(0, '../../')
      39  from Gnumed.pycommon import gmBorg
      40
      41
      42  _log = logging.getLogger('gm.tools')
      43
      44  # CAPitalization modes:
      45  (       CAPS_NONE,                                      # don't touch it
      46          CAPS_FIRST,                                     # CAP first char, leave rest as is
      47          CAPS_ALLCAPS,                           # CAP all chars
      48          CAPS_WORDS,                                     # CAP first char of every word
      49          CAPS_NAMES,                                     # CAP in a way suitable for names (tries to be smart)
      50          CAPS_FIRST_ONLY                         # CAP first char, lowercase the rest
      51  ) = range(6)
      52
      53
      54  u_currency_pound = '\u00A3'                             # Pound sign
      55  u_currency_sign = '\u00A4'                                      # generic currency sign
      56  u_currency_yen = '\u00A5'                                       # Yen sign
      57  u_right_double_angle_quote = '\u00AB'           # <<
      58  u_registered_trademark = '\u00AE'
      59  u_plus_minus = '\u00B1'
      60  u_superscript_one = '\u00B9'                            # ^1
      61  u_left_double_angle_quote = '\u00BB'            # >>
      62  u_one_quarter = '\u00BC'
      63  u_one_half = '\u00BD'
      64  u_three_quarters = '\u00BE'
      65  u_multiply = '\u00D7'                                           # x
      66  u_greek_ALPHA = '\u0391'
      67  u_greek_alpha = '\u03b1'
      68  u_greek_OMEGA = '\u03A9'
      69  u_greek_omega = '\u03c9'
      70  u_dagger = '\u2020'
      71  u_triangular_bullet = '\u2023'                                  # triangular bullet  (>)
      72  u_ellipsis = '\u2026'                                                   # ...
      73  u_euro = '\u20AC'                                                               # EURO sign
      74  u_numero = '\u2116'                                                             # No. / # sign
      75  u_down_left_arrow = '\u21B5'                                    # <-'
      76  u_left_arrow = '\u2190'                                                 # <--
      77  u_up_arrow = '\u2191'
      78  u_arrow2right = '\u2192'                                                # -->
      79  u_down_arrow = '\u2193'
      80  u_left_arrow_with_tail = '\u21a2'                               # <--<
      81  u_arrow2right_from_bar = '\u21a6'                               # |->
      82  u_arrow2right_until_vertical_bar = '\u21e5'             # -->|
      83  u_sum = '\u2211'                                                                # sigma
      84  u_almost_equal_to = '\u2248'                                    # approximately / nearly / roughly
      85  u_corresponds_to = '\u2258'
      86  u_infinity = '\u221E'
      87  u_arrow2right_until_vertical_bar2 = '\u2b72'    # -->|
      88
      89  u_diameter = '\u2300'
      90  u_checkmark_crossed_out = '\u237B'
      91  u_box_horiz_high = '\u23ba'
      92  u_box_vert_left = '\u23b8'
      93  u_box_vert_right = '\u23b9'
      94
      95  u_space_as_open_box = '\u2423'
      96
      97  u_box_horiz_single = '\u2500'                           # -
      98  u_box_vert_light = '\u2502'
      99  u_box_horiz_light_3dashes = '\u2504'            # ...
     100  u_box_vert_light_4dashes = '\u2506'
     101  u_box_horiz_4dashes = '\u2508'                          # ....
     102  u_box_T_right = '\u251c'                                        # |-
     103  u_box_T_left = '\u2524'                                         # -|
     104  u_box_T_down = '\u252c'
     105  u_box_T_up = '\u2534'
     106  u_box_plus = '\u253c'
     107  u_box_top_double = '\u2550'
     108  u_box_top_left_double_single = '\u2552'
     109  u_box_top_right_double_single = '\u2555'
     110  u_box_top_left_arc = '\u256d'
     111  u_box_top_right_arc = '\u256e'
     112  u_box_bottom_right_arc = '\u256f'
     113  u_box_bottom_left_arc = '\u2570'
     114  u_box_horiz_light_heavy = '\u257c'
     115  u_box_horiz_heavy_light = '\u257e'
     116
     117  u_skull_and_crossbones = '\u2620'
     118  u_caduceus = '\u2624'
     119  u_frowning_face = '\u2639'
     120  u_smiling_face = '\u263a'
     121  u_black_heart = '\u2665'
     122  u_female = '\u2640'
     123  u_male = '\u2642'
     124  u_male_female = '\u26a5'
     125  u_chain = '\u26d3'
     126
     127  u_checkmark_thin = '\u2713'
     128  u_checkmark_thick = '\u2714'
     129  u_heavy_greek_cross = '\u271a'
     130  u_arrow2right_thick = '\u2794'
     131  u_writing_hand = '\u270d'
     132  u_pencil_1 = '\u270e'
     133  u_pencil_2 = '\u270f'
     134  u_pencil_3 = '\u2710'
     135  u_latin_cross = '\u271d'
     136
     137  u_arrow2right_until_black_diamond = '\u291e'    # ->*
     138
     139  u_kanji_yen = '\u5186'                                                  # Yen kanji
     140  u_replacement_character = '\ufffd'
     141  u_link_symbol = '\u1f517'
     142
     143
     144  _kB = 1024
     145  _MB = 1024 * _kB
     146  _GB = 1024 * _MB
     147  _TB = 1024 * _GB
     148  _PB = 1024 * _TB
     149
     150
     151  _GM_TITLE_PREFIX = 'GMd'
     152
     153  #===========================================================================
     154 -def handle_uncaught_exception_console(t, v, tb):
     155
     156          print(".========================================================")
     157          print("| Unhandled exception caught !")
     158          print("| Type :", t)
     159          print("| Value:", v)
     160          print("`========================================================")
     161          _log.critical('unhandled exception caught', exc_info = (t,v,tb))
     162          sys.__excepthook__(t,v,tb)
     163
     164  #===========================================================================
     165  # path level operations
     166  #---------------------------------------------------------------------------
     167 -def mkdir(directory=None, mode=None):
     168          """Create directory.
     169
     170          - creates parent dirs if necessary
     171          - does not fail if directory exists
     172          <mode>: numeric, say 0o0700  for "-rwx------"
     173          """
     174          if os.path.isdir(directory):
     175                  if mode is None:
     176                          return True
     177
     178                  changed = False
     179                  old_umask = os.umask(0)
     180                  try:
     181                          # does not WORK !
     182                          #os.chmod(directory, mode, follow_symlinks = (os.chmod in os.supports_follow_symlinks)) # can't do better
     183                          os.chmod(directory, mode)
     184                          changed = True
     185                  finally:
     186                          os.umask(old_umask)
     187                  return changed
     188
     189          if mode is None:
     190                  os.makedirs(directory)
     191                  return True
     192
     193          old_umask = os.umask(0)
     194          try:
     195                  os.makedirs(directory, mode)
     196          finally:
     197                  os.umask(old_umask)
     198          return True
     199
     200  #---------------------------------------------------------------------------
     201 -def rmdir(directory):
     202          #-------------------------------
     203          def _on_rm_error(func, path, exc):
     204                  _log.error('error while shutil.rmtree(%s)', path, exc_info=exc)
     205                  return True
     206          #-------------------------------
     207          error_count = 0
     208          try:
     209                  shutil.rmtree(directory, False, _on_rm_error)
     210          except Exception:
     211                  _log.exception('cannot shutil.rmtree(%s)', directory)
     212                  error_count += 1
     213          return error_count
     214
     215  #---------------------------------------------------------------------------
     216 -def rm_dir_content(directory):
     217          _log.debug('cleaning out [%s]', directory)
     218          try:
     219                  items = os.listdir(directory)
     220          except OSError:
     221                  return False
     222          for item in items:
     223                  # attempt file/link removal and ignore (but log) errors
     224                  full_item = os.path.join(directory, item)
     225                  try:
     226                          os.remove(full_item)
     227                  except OSError:         # as per the docs, this is a directory
     228                          _log.debug('[%s] seems to be a subdirectory', full_item)
     229                          errors = rmdir(full_item)
     230                          if errors > 0:
     231                                  return False
     232                  except Exception:
     233                          _log.exception('cannot os.remove(%s) [a file or a link]', full_item)
     234                          return False
     235
     236          return True
     237
     238  #---------------------------------------------------------------------------
     239 -def mk_sandbox_dir(prefix=None, base_dir=None):
     240          if base_dir is None:
     241                  base_dir = gmPaths().tmp_dir
     242          else:
     243                  if not os.path.isdir(base_dir):
     244                          mkdir(base_dir, mode = 0o0700)  # (invoking user only)
     245          if prefix is None:
     246                  prefix = 'sndbx-'
     247          return tempfile.mkdtemp(prefix = prefix, suffix = '', dir = base_dir)
     248
     249  #---------------------------------------------------------------------------
     250 -def parent_dir(directory):
     251          return os.path.abspath(os.path.join(directory, '..'))
     252
     253  #---------------------------------------------------------------------------
     254 -def dirname_stem(directory):
     255          # /home/user/dir/ -> dir
     256          # /home/user/dir  -> dir
     257          return os.path.basename(os.path.normpath(directory))            # normpath removes trailing slashes if any
     258
     259  #---------------------------------------------------------------------------
     260 -def dir_is_empty(directory=None):
     261          try:
     262                  empty = (len(os.listdir(directory)) == 0)
     263          except OSError as exc:
     264                  if exc.errno != 2:      # no such file
     265                          raise
     266                  empty = None
     267          return empty
     268
     269  #---------------------------------------------------------------------------
     270 -def copy_tree_content(directory, target_directory):
     271          """Copy the *content* of <directory> *into* <target_directory>
     272             which is created if need be.
     273          """
     274          assert (directory is not None), 'source <directory> should not be None'
     275          _log.debug('copying content of [%s] into [%s]', directory, target_directory)
     276          try:
     277                  items = os.listdir(directory)
     278          except OSError:
     279                  return None
     280
     281          for item in items:
     282                  full_item = os.path.join(directory, item)
     283                  if os.path.isdir(full_item):
     284                          target_subdir = os.path.join(target_directory, item)
     285                          try:
     286                                  shutil.copytree(full_item, target_subdir)
     287                          except Exception:
     288                                  _log.exception('cannot copy subdir [%s]', full_item)
     289                                  return None
     290                  else:
     291                          try:
     292                                  shutil.copy2(full_item, target_directory)
     293                          except Exception:
     294                                  _log.exception('cannot copy file [%s]', full_item)
     295                                  return None
     296
     297          return target_directory
     298
     299  #---------------------------------------------------------------------------
     300  #---------------------------------------------------------------------------
     301 -class gmPaths(gmBorg.cBorg):
     302          """This class provides the following paths:
     303
     304          .home_dir                               user home
     305          .local_base_dir                 script installation dir
     306          .working_dir                    current dir
     307          .user_config_dir
     308          .system_config_dir
     309          .system_app_data_dir    (not writable)
     310          .tmp_dir                                instance-local
     311          .user_tmp_dir                   user-local (NOT per instance)
     312          .bytea_cache_dir                caches downloaded BYTEA data
     313          """
     314 -        def __init__(self, app_name=None, wx=None):
     315                  """Setup pathes.
     316
     317                  <app_name> will default to (name of the script - .py)
     318                  """
     319                  try:
     320                          self.already_inited
     321                          return
     322                  except AttributeError:
     323                          pass
     324
     325                  self.init_paths(app_name=app_name, wx=wx)
     326                  self.already_inited = True
     327
     328          #--------------------------------------
     329          # public API
     330          #--------------------------------------
     331 -        def init_paths(self, app_name=None, wx=None):
     332
     333                  if wx is None:
     334                          _log.debug('wxPython not available')
     335                  _log.debug('detecting paths directly')
     336
     337                  if app_name is None:
     338                          app_name, ext = os.path.splitext(os.path.basename(sys.argv[0]))
     339                          _log.info('app name detected as [%s]', app_name)
     340                  else:
     341                          _log.info('app name passed in as [%s]', app_name)
     342
     343                  # the user home, doesn't work in Wine so work around that
     344                  self.__home_dir = None
     345
     346                  # where the main script (the "binary") is installed
     347                  if getattr(sys, 'frozen', False):
     348                          _log.info('frozen app, installed into temporary path')
     349                          # this would find the path of *THIS* file
     350                          #self.local_base_dir = os.path.dirname(__file__)
     351                          # while this is documented on the web, the ${_MEIPASS2} does not exist
     352                          #self.local_base_dir = os.environ.get('_MEIPASS2')
     353                          # this is what Martin Zibricky <mzibr.public@gmail.com> told us to use
     354                          # when asking about this on pyinstaller@googlegroups.com
     355                          #self.local_base_dir = sys._MEIPASS
     356                          # however, we are --onedir, so we should look at sys.executable
     357                          # as per the pyinstaller manual
     358                          self.local_base_dir = os.path.dirname(sys.executable)
     359                  else:
     360                          self.local_base_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
     361
     362                  # the current working dir at the OS
     363                  self.working_dir = os.path.abspath(os.curdir)
     364
     365                  # user-specific config dir, usually below the home dir
     366                  mkdir(os.path.join(self.home_dir, '.%s' % app_name))
     367                  self.user_config_dir = os.path.join(self.home_dir, '.%s' % app_name)
     368
     369                  # system-wide config dir, usually below /etc/ under UN*X
     370                  try:
     371                          self.system_config_dir = os.path.join('/etc', app_name)
     372                  except ValueError:
     373                          #self.system_config_dir = self.local_base_dir
     374                          self.system_config_dir = self.user_config_dir
     375
     376                  # system-wide application data dir
     377                  try:
     378                          self.system_app_data_dir = os.path.join(sys.prefix, 'share', app_name)
     379                  except ValueError:
     380                          self.system_app_data_dir = self.local_base_dir
     381
     382                  # temporary directory
     383                  try:
     384                          self.__tmp_dir_already_set
     385                          _log.debug('temp dir already set')
     386                  except AttributeError:
     387                          _log.info('temp file prefix: %s', tempfile.gettempprefix())
     388                          _log.info('initial (user level) temp dir: %s', tempfile.gettempdir())
     389                          bytes_free = shutil.disk_usage(tempfile.gettempdir()).free
     390                          _log.info('free disk space for temp dir: %s (%s bytes)', size2str(size = bytes_free), bytes_free)
     391                          # $TMP/gnumed-$USER/
     392                          self.user_tmp_dir = os.path.join(tempfile.gettempdir(), '%s-%s' % (app_name, getpass.getuser()))
     393                          mkdir(self.user_tmp_dir, 0o700)
     394                          _log.info('intermediate (app+user level) temp dir: %s', self.user_tmp_dir)
     395                          # $TMP/gnumed-$USER/g-$UNIQUE/
     396                          tempfile.tempdir = self.user_tmp_dir # tell mkdtemp about intermediate dir
     397                          self.tmp_dir = tempfile.mkdtemp(prefix = 'g-') # will set tempfile.tempdir as side effect
     398                          _log.info('final (app instance level) temp dir: %s', tempfile.gettempdir())
     399
     400                  # BYTEA cache dir
     401                  cache_dir = os.path.join(self.user_tmp_dir, '.bytea_cache')
     402                  try:
     403                          stat = os.stat(cache_dir)
     404                          _log.warning('reusing BYTEA cache dir: %s', cache_dir)
     405                          _log.debug(stat)
     406                  except FileNotFoundError:
     407                          mkdir(cache_dir, mode = 0o0700)
     408                  self.bytea_cache_dir = cache_dir
     409
     410                  self.__log_paths()
     411                  if wx is None:
     412                          return True
     413
     414                  # retry with wxPython
     415                  _log.debug('re-detecting paths with wxPython')
     416
     417                  std_paths = wx.StandardPaths.Get()
     418                  _log.info('wxPython app name is [%s]', wx.GetApp().GetAppName())
     419
     420                  # user-specific config dir, usually below the home dir
     421                  mkdir(os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name))
     422                  self.user_config_dir = os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name)
     423
     424                  # system-wide config dir, usually below /etc/ under UN*X
     425                  try:
     426                          tmp = std_paths.GetConfigDir()
     427                          if not tmp.endswith(app_name):
     428                                  tmp = os.path.join(tmp, app_name)
     429                          self.system_config_dir = tmp
     430                  except ValueError:
     431                          # leave it at what it was from direct detection
     432                          pass
     433
     434                  # system-wide application data dir
     435                  # Robin attests that the following doesn't always
     436                  # give sane values on Windows, so IFDEF it
     437                  if 'wxMSW' in wx.PlatformInfo:
     438                          _log.warning('this platform (wxMSW) sometimes returns a broken value for the system-wide application data dir')
     439                  else:
     440                          try:
     441                                  self.system_app_data_dir = std_paths.GetDataDir()
     442                          except ValueError:
     443                                  pass
     444
     445                  self.__log_paths()
     446                  return True
     447
     448          #--------------------------------------
     449 -        def __log_paths(self):
     450                  _log.debug('sys.argv[0]: %s', sys.argv[0])
     451                  _log.debug('sys.executable: %s', sys.executable)
     452                  _log.debug('sys._MEIPASS: %s', getattr(sys, '_MEIPASS', '<not found>'))
     453                  _log.debug('os.environ["_MEIPASS2"]: %s', os.environ.get('_MEIPASS2', '<not found>'))
     454                  _log.debug('__file__ : %s', __file__)
     455                  _log.debug('local application base dir: %s', self.local_base_dir)
     456                  _log.debug('current working dir: %s', self.working_dir)
     457                  _log.debug('user home dir: %s', self.home_dir)
     458                  _log.debug('user-specific config dir: %s', self.user_config_dir)
     459                  _log.debug('system-wide config dir: %s', self.system_config_dir)
     460                  _log.debug('system-wide application data dir: %s', self.system_app_data_dir)
     461                  _log.debug('temporary dir (user): %s', self.user_tmp_dir)
     462                  _log.debug('temporary dir (instance): %s', self.tmp_dir)
     463                  _log.debug('temporary dir (tempfile.tempdir): %s', tempfile.tempdir)
     464                  _log.debug('temporary dir (tempfile.gettempdir()): %s', tempfile.gettempdir())
     465                  _log.debug('BYTEA cache dir: %s', self.bytea_cache_dir)
     466
     467          #--------------------------------------
     468          # properties
     469          #--------------------------------------
     470 -        def _set_user_config_dir(self, path):
     471                  if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
     472                          msg = '[%s:user_config_dir]: invalid path [%s]' % (self.__class__.__name__, path)
     473                          _log.error(msg)
     474                          raise ValueError(msg)
     475                  self.__user_config_dir = path
     476
     477 -        def _get_user_config_dir(self):
     478                  return self.__user_config_dir
     479
     480          user_config_dir = property(_get_user_config_dir, _set_user_config_dir)
     481          #--------------------------------------
     482 -        def _set_system_config_dir(self, path):
     483                  if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
     484                          msg = '[%s:system_config_dir]: invalid path [%s]' % (self.__class__.__name__, path)
     485                          _log.error(msg)
     486                          raise ValueError(msg)
     487                  self.__system_config_dir = path
     488
     489 -        def _get_system_config_dir(self):
     490                  return self.__system_config_dir
     491
     492          system_config_dir = property(_get_system_config_dir, _set_system_config_dir)
     493          #--------------------------------------
     494 -        def _set_system_app_data_dir(self, path):
     495                  if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
     496                          msg = '[%s:system_app_data_dir]: invalid path [%s]' % (self.__class__.__name__, path)
     497                          _log.error(msg)
     498                          raise ValueError(msg)
     499                  self.__system_app_data_dir = path
     500
     501 -        def _get_system_app_data_dir(self):
     502                  return self.__system_app_data_dir
     503
     504          system_app_data_dir = property(_get_system_app_data_dir, _set_system_app_data_dir)
     505          #--------------------------------------
     506 -        def _set_home_dir(self, path):
     507                  raise ValueError('invalid to set home dir')
     508
     509 -        def _get_home_dir(self):
     510                  if self.__home_dir is not None:
     511                          return self.__home_dir
     512
     513                  tmp = os.path.expanduser('~')
     514                  if tmp == '~':
     515                          _log.error('this platform does not expand ~ properly')
     516                          try:
     517                                  tmp = os.environ['USERPROFILE']
     518                          except KeyError:
     519                                  _log.error('cannot access $USERPROFILE in environment')
     520
     521                  if not (
     522                          os.access(tmp, os.R_OK)
     523                                  and
     524                          os.access(tmp, os.X_OK)
     525                                  and
     526                          os.access(tmp, os.W_OK)
     527                  ):
     528                          msg = '[%s:home_dir]: invalid path [%s]' % (self.__class__.__name__, tmp)
     529                          _log.error(msg)
     530                          raise ValueError(msg)
     531
     532                  self.__home_dir = tmp
     533                  return self.__home_dir
     534
     535          home_dir = property(_get_home_dir, _set_home_dir)
     536
     537          #--------------------------------------
     538 -        def _set_tmp_dir(self, path):
     539                  if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
     540                          msg = '[%s:tmp_dir]: invalid path [%s]' % (self.__class__.__name__, path)
     541                          _log.error(msg)
     542                          raise ValueError(msg)
     543                  _log.debug('previous temp dir: %s', tempfile.gettempdir())
     544                  self.__tmp_dir = path
     545                  tempfile.tempdir = self.__tmp_dir
     546                  _log.debug('new temp dir: %s', tempfile.gettempdir())
     547                  self.__tmp_dir_already_set = True
     548
     549 -        def _get_tmp_dir(self):
     550                  return self.__tmp_dir
     551
     552          tmp_dir = property(_get_tmp_dir, _set_tmp_dir)
     553
     554  #===========================================================================
     555  # file related tools
     556  #---------------------------------------------------------------------------
     557 -def recode_file(source_file=None, target_file=None, source_encoding='utf8', target_encoding=None, base_dir=None, error_mode='replace'):
     558          if target_encoding is None:
     559                  return source_file
     560          if target_encoding == source_encoding:
     561                  return source_file
     562          if target_file is None:
     563                  target_file = get_unique_filename (
     564                          prefix = '%s-%s_%s-' % (fname_stem(source_file), source_encoding, target_encoding),
     565                          suffix = fname_extension(source_file, '.txt'),
     566                          tmp_dir = base_dir
     567                  )
     568
     569          _log.debug('[%s] -> [%s] (%s -> %s)', source_encoding, target_encoding, source_file, target_file)
     570
     571          in_file = io.open(source_file, mode = 'rt', encoding = source_encoding)
     572          out_file = io.open(target_file, mode = 'wt', encoding = target_encoding, errors = error_mode)
     573          for line in in_file:
     574                  out_file.write(line)
     575          out_file.close()
     576          in_file.close()
     577
     578          return target_file
     579
     580  #---------------------------------------------------------------------------
     581 -def unzip_archive(archive_name, target_dir=None, remove_archive=False):
     582          _log.debug('unzipping [%s] -> [%s]', archive_name, target_dir)
     583          success = False
     584          try:
     585                  with zipfile.ZipFile(archive_name) as archive:
     586                          archive.extractall(target_dir)
     587                  success = True
     588          except Exception:
     589                  _log.exception('cannot unzip')
     590                  return False
     591          if remove_archive:
     592                  remove_file(archive_name)
     593          return success
     594
     595  #---------------------------------------------------------------------------
     596 -def remove_file(filename, log_error=True, force=False):
     597          if not os.path.lexists(filename):
     598                  return True
     599
     600          # attempt file removal and ignore (but log) errors
     601          try:
     602                  os.remove(filename)
     603                  return True
     604
     605          except Exception:
     606                  if log_error:
     607                          _log.exception('cannot os.remove(%s)', filename)
     608
     609          if force:
     610                  tmp_name = get_unique_filename(tmp_dir = fname_dir(filename))
     611                  _log.debug('attempting os.replace(%s -> %s)', filename, tmp_name)
     612                  try:
     613                          os.replace(filename, tmp_name)
     614                          return True
     615
     616                  except Exception:
     617                          if log_error:
     618                                  _log.exception('cannot os.remove(%s)', filename)
     619
     620          return False
     621
     622  #---------------------------------------------------------------------------
     623 -def file2md5(filename=None, return_hex=True):
     624          blocksize = 2**10 * 128                 # 128k, since md5 uses 128 byte blocks
     625          _log.debug('md5(%s): <%s> byte blocks', filename, blocksize)
     626          f = io.open(filename, mode = 'rb')
     627          md5 = hashlib.md5()
     628          while True:
     629                  data = f.read(blocksize)
     630                  if not data:
     631                          break
     632                  md5.update(data)
     633          f.close()
     634          _log.debug('md5(%s): %s', filename, md5.hexdigest())
     635          if return_hex:
     636                  return md5.hexdigest()
     637
     638          return md5.digest()
     639
     640  #---------------------------------------------------------------------------
     641 -def file2chunked_md5(filename=None, chunk_size=500*_MB):
     642          _log.debug('chunked_md5(%s, chunk_size=%s bytes)', filename, chunk_size)
     643          md5_concat = ''
     644          f = open(filename, 'rb')
     645          while True:
     646                  md5 = hashlib.md5()
     647                  data = f.read(chunk_size)
     648                  if not data:
     649                          break
     650                  md5.update(data)
     651                  md5_concat += md5.hexdigest()
     652          f.close()
     653          md5 = hashlib.md5()
     654          md5.update(md5_concat)
     655          hex_digest = md5.hexdigest()
     656          _log.debug('md5("%s"): %s', md5_concat, hex_digest)
     657          return hex_digest
     658
     659  #---------------------------------------------------------------------------
     660  default_csv_reader_rest_key = 'list_of_values_of_unknown_fields'
     661
     662 -def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, encoding='utf-8', **kwargs):
     663          try:
     664                  is_dict_reader = kwargs['dict']
     665                  del kwargs['dict']
     666          except KeyError:
     667                  is_dict_reader = False
     668
     669          if is_dict_reader:
     670                  kwargs['restkey'] = default_csv_reader_rest_key
     671                  return csv.DictReader(unicode_csv_data, dialect=dialect, **kwargs)
     672          return csv.reader(unicode_csv_data, dialect=dialect, **kwargs)
     673
     674
     675
     676
     677 -def old_unicode2charset_encoder(unicode_csv_data, encoding='utf-8'):
     678          for line in unicode_csv_data:
     679                  yield line.encode(encoding)
     680
     681  #def utf_8_encoder(unicode_csv_data):
     682  #       for line in unicode_csv_data:
     683  #               yield line.encode('utf-8')
     684
     685 -def old_unicode_csv_reader(unicode_csv_data, dialect=csv.excel, encoding='utf-8', **kwargs):
     686
     687          # csv.py doesn't do Unicode; encode temporarily as UTF-8:
     688          try:
     689                  is_dict_reader = kwargs['dict']
     690                  del kwargs['dict']
     691                  if is_dict_reader is not True:
     692                          raise KeyError
     693                  kwargs['restkey'] = default_csv_reader_rest_key
     694                  csv_reader = csv.DictReader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs)
     695          except KeyError:
     696                  is_dict_reader = False
     697                  csv_reader = csv.reader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs)
     698
     699          for row in csv_reader:
     700                  # decode ENCODING back to Unicode, cell by cell:
     701                  if is_dict_reader:
     702                          for key in row.keys():
     703                                  if key == default_csv_reader_rest_key:
     704                                          old_data = row[key]
     705                                          new_data = []
     706                                          for val in old_data:
     707                                                  new_data.append(str(val, encoding))
     708                                          row[key] = new_data
     709                                          if default_csv_reader_rest_key not in csv_reader.fieldnames:
     710                                                  csv_reader.fieldnames.append(default_csv_reader_rest_key)
     711                                  else:
     712                                          row[key] = str(row[key], encoding)
     713                          yield row
     714                  else:
     715                          yield [ str(cell, encoding) for cell in row ]
     716                          #yield [str(cell, 'utf-8') for cell in row]
     717
     718  #---------------------------------------------------------------------------
     719 -def fname_sanitize(filename):
     720          """Normalizes unicode, removes non-alpha characters, converts spaces to underscores."""
     721
     722          dir_part, name_part = os.path.split(filename)
     723          if name_part == '':
     724                  return filename
     725
     726          import unicodedata
     727          name_part = unicodedata.normalize('NFKD', name_part)
     728          # remove everything not in group []
     729          name_part = regex.sub (
     730                  '[^.\w\s[\]()%§+-]',
     731                  '',
     732                  name_part,
     733                  flags = regex.UNICODE
     734          ).strip()
     735          # translate whitespace to underscore
     736          name_part = regex.sub (
     737                  '\s+',
     738                  '_',
     739                  name_part,
     740                  flags = regex.UNICODE
     741          )
     742          return os.path.join(dir_part, name_part)
     743
     744  #---------------------------------------------------------------------------
     745 -def fname_stem(filename):
     746          """/home/user/dir/filename.ext -> filename"""
     747          return os.path.splitext(os.path.basename(filename))[0]
     748
     749  #---------------------------------------------------------------------------
     750 -def fname_stem_with_path(filename):
     751          """/home/user/dir/filename.ext -> /home/user/dir/filename"""
     752          return os.path.splitext(filename)[0]
     753
     754  #---------------------------------------------------------------------------
     755 -def fname_extension(filename=None, fallback=None):
     756          """     /home/user/dir/filename.ext -> .ext
     757                  '' or '.' -> fallback if any else ''
     758          """
     759          ext = os.path.splitext(filename)[1]
     760          if ext.strip() not in ['.', '']:
     761                  return ext
     762          if fallback is None:
     763                  return ''
     764          return fallback
     765
     766  #---------------------------------------------------------------------------
     767 -def fname_dir(filename):
     768          # /home/user/dir/filename.ext -> /home/user/dir
     769          return os.path.split(filename)[0]
     770
     771  #---------------------------------------------------------------------------
     772 -def fname_from_path(filename):
     773          # /home/user/dir/filename.ext -> filename.ext
     774          return os.path.split(filename)[1]
     775
     776  #---------------------------------------------------------------------------
     777 -def get_unique_filename(prefix=None, suffix=None, tmp_dir=None, include_timestamp=False):
     778          """This function has a race condition between
     779                          its file.close()
     780             and actually
     781                          using the filename in callers.
     782
     783          The file will NOT exist after calling this function.
     784          """
     785          if tmp_dir is None:
     786                  gmPaths()               # setup tmp dir if necessary
     787          else:
     788                  if (
     789                          not os.access(tmp_dir, os.F_OK)
     790                                  or
     791                          not os.access(tmp_dir, os.X_OK | os.W_OK)
     792                  ):
     793                          _log.warning('cannot os.access() temporary dir [%s], using system default', tmp_dir)
     794                          tmp_dir = None
     795
     796          if include_timestamp:
     797                  ts = pydt.datetime.now().strftime('%m%d-%H%M%S-')
     798          else:
     799                  ts = ''
     800
     801          kwargs = {
     802                  'dir': tmp_dir,
     803                  #  make sure file gets deleted as soon as
     804                  # .close()d so we can "safely" open it again
     805                  'delete': True
     806          }
     807
     808          if prefix is None:
     809                  kwargs['prefix'] = 'gm-%s' % ts
     810          else:
     811                  kwargs['prefix'] = prefix + ts
     812
     813          if suffix in [None, '']:
     814                  kwargs['suffix'] = '.tmp'
     815          else:
     816                  if not suffix.startswith('.'):
     817                          suffix = '.' + suffix
     818                  kwargs['suffix'] = suffix
     819
     820          f = tempfile.NamedTemporaryFile(**kwargs)
     821          filename = f.name
     822          f.close()
     823
     824          return filename
     825
     826  #---------------------------------------------------------------------------
     827 -def __make_symlink_on_windows(physical_name, link_name):
     828          import ctypes
     829          #windows_create_symlink = ctypes.windll.kernel32.CreateSymbolicLinkW
     830          kernel32 = ctype.WinDLL('kernel32', use_last_error = True)
     831          windows_create_symlink = kernel32.CreateSymbolicLinkW
     832          windows_create_symlink.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
     833          windows_create_symlink.restype = ctypes.c_ubyte
     834          if os.path.isdir(physical_name):
     835                  flags = 1
     836          else:
     837                  flags = 0
     838          ret_code = windows_create_symlink(link_name, physical_name.replace('/', '\\'), flags)
     839          _log.debug('ctypes.windll.kernel32.CreateSymbolicLinkW() [%s] exit code: %s', windows_create_symlink, ret_code)
     840          if ret_code == 0:
     841                  raise ctypes.WinError()
     842          return ret_code
     843
     844  #---------------------------------------------------------------------------
     845 -def mklink(physical_name, link_name, overwrite=False):
     846
     847          _log.debug('creating symlink (overwrite = %s):', overwrite)
     848          _log.debug('link [%s] =>', link_name)
     849          _log.debug('=> physical [%s]', physical_name)
     850
     851          if os.path.exists(link_name):
     852                  _log.debug('link exists')
     853                  if overwrite:
     854                          return True
     855                  return False
     856
     857          try:
     858                  os.symlink(physical_name, link_name)
     859          except (AttributeError, NotImplementedError):
     860                  _log.debug('this Python does not have os.symlink(), trying via ctypes')
     861                  __make_symlink_on_windows(physical_name, link_name)
     862          except PermissionError:
     863                  _log.exception('cannot create link')
     864                  return False
     865          #except OSError:
     866          #       unpriviledged on Windows
     867          return True
     868
     869  #===========================================================================
     870 -def import_module_from_directory(module_path=None, module_name=None, always_remove_path=False):
     871          """Import a module from any location."""
     872
     873          _log.debug('CWD: %s', os.getcwd())
     874
     875          remove_path = always_remove_path or False
     876          if module_path not in sys.path:
     877                  _log.info('appending to sys.path: [%s]' % module_path)
     878                  sys.path.append(module_path)
     879                  remove_path = True
     880
     881          _log.debug('will remove import path: %s', remove_path)
     882
     883          if module_name.endswith('.py'):
     884                  module_name = module_name[:-3]
     885
     886          try:
     887                  module = __import__(module_name)
     888          except Exception:
     889                  _log.exception('cannot __import__() module [%s] from [%s]' % (module_name, module_path))
     890                  while module_path in sys.path:
     891                          sys.path.remove(module_path)
     892                  raise
     893
     894          _log.info('imported module [%s] as [%s]' % (module_name, module))
     895          if remove_path:
     896                  while module_path in sys.path:
     897                          sys.path.remove(module_path)
     898
     899          return module
     900
     901  #===========================================================================
     902  # text related tools
     903  #---------------------------------------------------------------------------
     904 -def size2str(size=0, template='%s'):
     905          if size == 1:
     906                  return template % _('1 Byte')
     907          if size < 10 * _kB:
     908                  return template % _('%s Bytes') % size
     909          if size < _MB:
     910                  return template % '%.1f kB' % (float(size) / _kB)
     911          if size < _GB:
     912                  return template % '%.1f MB' % (float(size) / _MB)
     913          if size < _TB:
     914                  return template % '%.1f GB' % (float(size) / _GB)
     915          if size < _PB:
     916                  return template % '%.1f TB' % (float(size) / _TB)
     917          return template % '%.1f PB' % (float(size) / _PB)
     918
     919  #---------------------------------------------------------------------------
     920 -def bool2subst(boolean=None, true_return=True, false_return=False, none_return=None):
     921          if boolean is None:
     922                  return none_return
     923          if boolean:
     924                  return true_return
     925          if not boolean:
     926                  return false_return
     927          raise ValueError('bool2subst(): <boolean> arg must be either of True, False, None')
     928
     929  #---------------------------------------------------------------------------
     930 -def bool2str(boolean=None, true_str='True', false_str='False'):
     931          return bool2subst (
     932                  boolean = bool(boolean),
     933                  true_return = true_str,
     934                  false_return = false_str
     935          )
     936
     937  #---------------------------------------------------------------------------
     938 -def none_if(value=None, none_equivalent=None, strip_string=False):
     939          """Modelled after the SQL NULLIF function."""
     940          if value is None:
     941                  return None
     942
     943          if strip_string:
     944                  stripped = value.strip()
     945          else:
     946                  stripped = value
     947          if stripped == none_equivalent:
     948                  return None
     949
     950          return value
     951
     952  #---------------------------------------------------------------------------
     953 -def coalesce(value2test=None, return_instead=None, template4value=None, template4instead=None, none_equivalents=None, function4value=None, value2return=None):
     954          """Modelled after the SQL coalesce function.
     955
     956          To be used to simplify constructs like:
     957
     958                  if value2test is None (or in none_equivalents):
     959                          value = (template4instead % return_instead) or return_instead
     960                  else:
     961                          value = (template4value % value2test) or value2test
     962                  print value
     963
     964          @param value2test: the value to be tested for <None>
     965
     966          @param return_instead: the value to be returned if <value2test> *is* None
     967
     968          @param template4value: if <value2test> is returned, replace the value into this template, must contain one <%s>
     969
     970          @param template4instead: if <return_instead> is returned, replace the value into this template, must contain one <%s>
     971
     972          @param value2return: a *value* to return if <value2test> is NOT None, AND there's no <template4value>
     973
     974          example:
     975                  function4value = ('strftime', '%Y-%m-%d')
     976
     977          Ideas:
     978                  - list of return_insteads: initial, [return_instead, template], [return_instead, template], [return_instead, template], template4value, ...
     979          """
     980          if none_equivalents is None:
     981                  none_equivalents = [None]
     982
     983          if value2test in none_equivalents:
     984                  if template4instead is None:
     985                          return return_instead
     986                  return template4instead % return_instead
     987
     988          # at this point, value2test was not equivalent to None
     989
     990          # 1) explicit value to return supplied ?
     991          if value2return is not None:
     992                  return value2return
     993
     994          value2return = value2test
     995          # 2) function supplied to be applied to the value ?
     996          if function4value is not None:
     997                  funcname, args = function4value
     998                  func = getattr(value2test, funcname)
     999                  value2return = func(args)
    1000
    1001          # 3) template supplied to be applied to the value ?
    1002          if template4value is None:
    1003                  return value2return
    1004
    1005          try:
    1006                  return template4value % value2return
    1007          except TypeError:
    1008  #       except (TypeError, ValueError):
    1009                  # this should go, actually, only needed because "old" calls
    1010                  # to coalesce will still abuse template4value as explicit value2return,
    1011                  # relying on the replacement to above to fail
    1012                  if hasattr(_log, 'log_stack_trace'):
    1013                          _log.log_stack_trace(message = 'deprecated use of <template4value> for <value2return>')
    1014                  else:
    1015                          _log.error('deprecated use of <template4value> for <value2return>')
    1016                          _log.error(locals())
    1017                  return template4value
    1018
    1019  #---------------------------------------------------------------------------
    1020 -def __cap_name(match_obj=None):
    1021          val = match_obj.group(0).lower()
    1022          if val in ['von', 'van', 'de', 'la', 'l', 'der', 'den']:                        # FIXME: this needs to expand, configurable ?
    1023                  return val
    1024          buf = list(val)
    1025          buf[0] = buf[0].upper()
    1026          for part in ['mac', 'mc', 'de', 'la']:
    1027                  if len(val) > len(part) and val[:len(part)] == part:
    1028                          buf[len(part)] = buf[len(part)].upper()
    1029          return ''.join(buf)
    1030
    1031  #---------------------------------------------------------------------------
    1032 -def capitalize(text=None, mode=CAPS_NAMES):
    1033          """Capitalize the first character but leave the rest alone.
    1034
    1035          Note that we must be careful about the locale, this may
    1036          have issues ! However, for UTF strings it should just work.
    1037          """
    1038          if (mode is None) or (mode == CAPS_NONE):
    1039                  return text
    1040
    1041          if len(text) == 0:
    1042                  return text
    1043
    1044          if mode == CAPS_FIRST:
    1045                  if len(text) == 1:
    1046                          return text[0].upper()
    1047                  return text[0].upper() + text[1:]
    1048
    1049          if mode == CAPS_ALLCAPS:
    1050                  return text.upper()
    1051
    1052          if mode == CAPS_FIRST_ONLY:
    1053  #               if len(text) == 1:
    1054  #                       return text[0].upper()
    1055                  return text[0].upper() + text[1:].lower()
    1056
    1057          if mode == CAPS_WORDS:
    1058                  #return regex.sub(ur'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text)
    1059                  return regex.sub(r'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text)
    1060
    1061          if mode == CAPS_NAMES:
    1062                  #return regex.sub(r'\w+', __cap_name, text)
    1063                  return capitalize(text=text, mode=CAPS_FIRST)           # until fixed
    1064
    1065          print("ERROR: invalid capitalization mode: [%s], leaving input as is" % mode)
    1066          return text
    1067
    1068  #---------------------------------------------------------------------------
    1069 -def input2decimal(initial=None):
    1070
    1071          if isinstance(initial, decimal.Decimal):
    1072                  return True, initial
    1073
    1074          val = initial
    1075
    1076          # float ? -> to string first
    1077          if type(val) == type(float(1.4)):
    1078                  val = str(val)
    1079
    1080          # string ? -> "," to "."
    1081          if isinstance(val, str):
    1082                  val = val.replace(',', '.', 1)
    1083                  val = val.strip()
    1084
    1085          try:
    1086                  d = decimal.Decimal(val)
    1087                  return True, d
    1088          except (TypeError, decimal.InvalidOperation):
    1089                  return False, val
    1090
    1091  #---------------------------------------------------------------------------
    1092 -def input2int(initial=None, minval=None, maxval=None):
    1093
    1094          val = initial
    1095
    1096          # string ? -> "," to "."
    1097          if isinstance(val, str):
    1098                  val = val.replace(',', '.', 1)
    1099                  val = val.strip()
    1100
    1101          try:
    1102                  int_val = int(val)
    1103          except (TypeError, ValueError):
    1104                  _log.exception('int(%s) failed', val)
    1105                  return False, initial
    1106
    1107          if minval is not None:
    1108                  if int_val < minval:
    1109                          _log.debug('%s < min (%s)', val, minval)
    1110                          return False, initial
    1111          if maxval is not None:
    1112                  if int_val > maxval:
    1113                          _log.debug('%s > max (%s)', val, maxval)
    1114                          return False, initial
    1115
    1116          return True, int_val
    1117
    1118  #---------------------------------------------------------------------------
    1119 -def strip_prefix(text, prefix, remove_repeats=False, remove_whitespace=False):
    1120          if remove_whitespace:
    1121                  text = text.lstrip()
    1122          if not text.startswith(prefix):
    1123                  return text
    1124
    1125          text = text.replace(prefix, '', 1)
    1126          if not remove_repeats:
    1127                  if remove_whitespace:
    1128                          return text.lstrip()
    1129                  return text
    1130
    1131          return strip_prefix(text, prefix, remove_repeats = True, remove_whitespace = remove_whitespace)
    1132
    1133  #---------------------------------------------------------------------------
    1134 -def decorate_window_title(title):
    1135          if title.startswith(_GM_TITLE_PREFIX):
    1136                  return title
    1137          return '%s: %s' % (
    1138                  _GM_TITLE_PREFIX,
    1139                  title.lstrip()
    1140          )
    1141
    1142  #---------------------------------------------------------------------------
    1143 -def undecorate_window_title(title):
    1144          return strip_prefix(title, _GM_TITLE_PREFIX + ':', remove_repeats = True, remove_whitespace = True)
    1145
    1146  #---------------------------------------------------------------------------
    1147 -def strip_suffix(text, suffix, remove_repeats=False, remove_whitespace=False):
    1148          suffix_len = len(suffix)
    1149          if remove_repeats:
    1150                  if remove_whitespace:
    1151                          while text.rstrip().endswith(suffix):
    1152                                  text = text.rstrip()[:-suffix_len].rstrip()
    1153                          return text
    1154                  while text.endswith(suffix):
    1155                          text = text[:-suffix_len]
    1156                  return text
    1157          if remove_whitespace:
    1158                  return text.rstrip()[:-suffix_len].rstrip()
    1159          return text[:-suffix_len]
    1160
    1161  #---------------------------------------------------------------------------
    1162 -def strip_leading_empty_lines(lines=None, text=None, eol='\n', return_list=True):
    1163          if lines is None:
    1164                  lines = text.split(eol)
    1165
    1166          while True:
    1167                  if lines[0].strip(eol).strip() != '':
    1168                          break
    1169                  lines = lines[1:]
    1170
    1171          if return_list:
    1172                  return lines
    1173
    1174          return eol.join(lines)
    1175
    1176  #---------------------------------------------------------------------------
    1177 -def strip_trailing_empty_lines(lines=None, text=None, eol='\n', return_list=True):
    1178          if lines is None:
    1179                  lines = text.split(eol)
    1180
    1181          while True:
    1182                  if lines[-1].strip(eol).strip() != '':
    1183                          break
    1184                  lines = lines[:-1]
    1185
    1186          if return_list:
    1187                  return lines
    1188
    1189          return eol.join(lines)
    1190
    1191  #---------------------------------------------------------------------------
    1192 -def strip_empty_lines(lines=None, text=None, eol='\n', return_list=True):
    1193          return strip_trailing_empty_lines (
    1194                  lines = strip_leading_empty_lines(lines = lines, text = text, eol = eol, return_list = True),
    1195                  text = None,
    1196                  eol = eol,
    1197                  return_list = return_list
    1198          )
    1199
    1200  #---------------------------------------------------------------------------
    1201 -def list2text(lines, initial_indent='', subsequent_indent='', eol='\n', strip_leading_empty_lines=True, strip_trailing_empty_lines=True, strip_trailing_whitespace=True, max_line_width=None):
    1202
    1203          if len(lines) == 0:
    1204                  return ''
    1205
    1206          if strip_leading_empty_lines:
    1207                  lines = strip_leading_empty_lines(lines = lines, eol = eol, return_list = True)
    1208
    1209          if strip_trailing_empty_lines:
    1210                  lines = strip_trailing_empty_lines(lines = lines, eol = eol, return_list = True)
    1211
    1212          if strip_trailing_whitespace:
    1213                  lines = [ l.rstrip() for l in lines ]
    1214
    1215          if max_line_width is not None:
    1216                  wrapped_lines = []
    1217                  for l in lines:
    1218                          wrapped_lines.extend(wrap(l, max_line_width).split('\n'))
    1219                  lines = wrapped_lines
    1220
    1221          indented_lines = [initial_indent + lines[0]]
    1222          indented_lines.extend([ subsequent_indent + l for l in lines[1:] ])
    1223
    1224          return eol.join(indented_lines)
    1225
    1226  #---------------------------------------------------------------------------
    1227 -def wrap(text=None, width=None, initial_indent='', subsequent_indent='', eol='\n'):
    1228          """A word-wrap function that preserves existing line breaks
    1229          and most spaces in the text. Expects that existing line
    1230          breaks are posix newlines (\n).
    1231          """
    1232          if width is None:
    1233                  return text
    1234
    1235          wrapped = initial_indent + functools.reduce (
    1236                  lambda line, word, width=width: '%s%s%s' % (
    1237                          line,
    1238                          ' \n'[(len(line) - line.rfind('\n') - 1 + len(word.split('\n',1)[0]) >= width)],
    1239                          word
    1240                  ),
    1241                  text.split(' ')
    1242          )
    1243          if subsequent_indent != '':
    1244                  wrapped = ('\n%s' % subsequent_indent).join(wrapped.split('\n'))
    1245          if eol != '\n':
    1246                  wrapped = wrapped.replace('\n', eol)
    1247          return wrapped
    1248
    1249  #---------------------------------------------------------------------------
    1250 -def unwrap(text=None, max_length=None, strip_whitespace=True, remove_empty_lines=True, line_separator = ' // '):
    1251
    1252          text = text.replace('\r', '')
    1253          lines = text.split('\n')
    1254          text = ''
    1255          for line in lines:
    1256
    1257                  if strip_whitespace:
    1258                          line = line.strip().strip('\t').strip()
    1259
    1260                  if remove_empty_lines:
    1261                          if line == '':
    1262                                  continue
    1263
    1264                  text += ('%s%s' % (line, line_separator))
    1265
    1266          text = text.rstrip(line_separator)
    1267
    1268          if max_length is not None:
    1269                  text = text[:max_length]
    1270
    1271          text = text.rstrip(line_separator)
    1272
    1273          return text
    1274
    1275  #---------------------------------------------------------------------------
    1276 -def shorten_text(text=None, max_length=None):
    1277
    1278          if len(text) <= max_length:
    1279                  return text
    1280
    1281          return text[:max_length-1] + u_ellipsis
    1282
    1283  #---------------------------------------------------------------------------
    1284 -def shorten_words_in_line(text=None, max_length=None, min_word_length=None, ignore_numbers=True, ellipsis=u_ellipsis):
    1285          if text is None:
    1286                  return None
    1287          if max_length is None:
    1288                  max_length = len(text)
    1289          else:
    1290                  if len(text) <= max_length:
    1291                          return text
    1292          old_words = regex.split('\s+', text, flags = regex.UNICODE)
    1293          no_old_words = len(old_words)
    1294          max_word_length = max(min_word_length, (max_length // no_old_words))
    1295          words = []
    1296          for word in old_words:
    1297                  if len(word) <= max_word_length:
    1298                          words.append(word)
    1299                          continue
    1300                  if ignore_numbers:
    1301                          tmp = word.replace('-', '').replace('+', '').replace('.', '').replace(',', '').replace('/', '').replace('&', '').replace('*', '')
    1302                          if tmp.isdigit():
    1303                                  words.append(word)
    1304                                  continue
    1305                  words.append(word[:max_word_length] + ellipsis)
    1306          return ' '.join(words)
    1307
    1308  #---------------------------------------------------------------------------
    1309 -def xml_escape_string(text=None):
    1310          """check for special XML characters and transform them"""
    1311          return xml_tools.escape(text)
    1312
    1313  #---------------------------------------------------------------------------
    1314 -def tex_escape_string(text=None, replace_known_unicode=True, replace_eol=False, keep_visual_eol=False):
    1315          """Check for special TeX characters and transform them.
    1316
    1317                  replace_eol:
    1318                          replaces "\n" with "\\newline"
    1319                  keep_visual_eol:
    1320                          replaces "\n" with "\\newline \n" such that
    1321                          both LaTeX will know to place a line break
    1322                          at this point as well as the visual formatting
    1323                          is preserved in the LaTeX source (think multi-
    1324                          row table cells)
    1325          """
    1326          text = text.replace('\\', '\\textbackslash')                    # requires \usepackage{textcomp} in LaTeX source
    1327          text = text.replace('^', '\\textasciicircum')
    1328          text = text.replace('~', '\\textasciitilde')
    1329
    1330          text = text.replace('{', '\\{')
    1331          text = text.replace('}', '\\}')
    1332          text = text.replace('%', '\\%')
    1333          text = text.replace('&', '\\&')
    1334          text = text.replace('#', '\\#')
    1335          text = text.replace('$', '\\$')
    1336          text = text.replace('_', '\\_')
    1337          if replace_eol:
    1338                  if keep_visual_eol:
    1339                          text = text.replace('\n', '\\newline \n')
    1340                  else:
    1341                          text = text.replace('\n', '\\newline ')
    1342
    1343          if replace_known_unicode:
    1344                  # this should NOT be replaced for Xe(La)Tex
    1345                  text = text.replace(u_euro, '\\EUR')            # requires \usepackage{textcomp} in LaTeX source
    1346                  text = text.replace(u_sum, '$\\Sigma$')
    1347
    1348          return text
    1349
    1350  #---------------------------------------------------------------------------
    1351 -def rst2latex_snippet(rst_text):
    1352          global du_core
    1353          if du_core is None:
    1354                  try:
    1355                          from docutils import core as du_core
    1356                  except ImportError:
    1357                          _log.warning('cannot turn ReST into LaTeX: docutils not installed')
    1358                          return tex_escape_string(text = rst_text)
    1359
    1360          parts = du_core.publish_parts (
    1361                  source = rst_text.replace('\\', '\\\\'),
    1362                  source_path = '<internal>',
    1363                  writer_name = 'latex',
    1364                  #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex',
    1365                  settings_overrides = {
    1366                          'input_encoding': 'unicode'             # un-encoded unicode
    1367                  },
    1368                  enable_exit_status = True                       # how to use ?
    1369          )
    1370          return parts['body']
    1371
    1372  #---------------------------------------------------------------------------
    1373 -def rst2html(rst_text, replace_eol=False, keep_visual_eol=False):
    1374          global du_core
    1375          if du_core is None:
    1376                  try:
    1377                          from docutils import core as du_core
    1378                  except ImportError:
    1379                          _log.warning('cannot turn ReST into HTML: docutils not installed')
    1380                          return html_escape_string(text = rst_text, replace_eol=False, keep_visual_eol=False)
    1381
    1382          parts = du_core.publish_parts (
    1383                  source = rst_text.replace('\\', '\\\\'),
    1384                  source_path = '<internal>',
    1385                  writer_name = 'latex',
    1386                  #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex',
    1387                  settings_overrides = {
    1388                          'input_encoding': 'unicode'             # un-encoded unicode
    1389                  },
    1390                  enable_exit_status = True                       # how to use ?
    1391          )
    1392          return parts['body']
    1393
    1394  #---------------------------------------------------------------------------
    1395 -def xetex_escape_string(text=None):
    1396          # a web search did not reveal anything else for Xe(La)Tex
    1397          # as opposed to LaTeX, except true unicode chars
    1398          return tex_escape_string(text = text, replace_known_unicode = False)
    1399
    1400  #---------------------------------------------------------------------------
    1401  __html_escape_table = {
    1402          "&": "&amp;",
    1403          '"': "&quot;",
    1404          "'": "&apos;",
    1405          ">": "&gt;",
    1406          "<": "&lt;",
    1407  }
    1408
    1409 -def html_escape_string(text=None, replace_eol=False, keep_visual_eol=False):
    1410          text = ''.join(__html_escape_table.get(char, char) for char in text)
    1411          if replace_eol:
    1412                  if keep_visual_eol:
    1413                          text = text.replace('\n', '<br>\n')
    1414                  else:
    1415                          text = text.replace('\n', '<br>')
    1416          return text
    1417
    1418  #---------------------------------------------------------------------------
    1419 -def dict2json(obj):
    1420          return json.dumps(obj, default = json_serialize)
    1421
    1422  #---------------------------------------------------------------------------
    1423 -def json_serialize(obj):
    1424          if isinstance(obj, pydt.datetime):
    1425                  return obj.isoformat()
    1426          raise TypeError('cannot json_serialize(%s)' % type(obj))
    1427
    1428  #---------------------------------------------------------------------------
    1429  #---------------------------------------------------------------------------
    1430 -def compare_dict_likes(d1, d2, title1=None, title2=None):
    1431          _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title1, '', '"%s" '), type(d1), coalesce(title2, '', '"%s" '), type(d2))
    1432          try:
    1433                  d1 = dict(d1)
    1434          except TypeError:
    1435                  pass
    1436          try:
    1437                  d2 = dict(d2)
    1438          except TypeError:
    1439                  pass
    1440          keys_d1 = frozenset(d1.keys())
    1441          keys_d2 = frozenset(d2.keys())
    1442          different = False
    1443          if len(keys_d1) != len(keys_d2):
    1444                  _log.info('different number of keys: %s vs %s', len(keys_d1), len(keys_d2))
    1445                  different = True
    1446          for key in keys_d1:
    1447                  if key in keys_d2:
    1448                          if type(d1[key]) != type(d2[key]):
    1449                                  _log.info('%25.25s: type(dict1) = %s = >>>%s<<<' % (key, type(d1[key]), d1[key]))
    1450                                  _log.info('%25.25s  type(dict2) = %s = >>>%s<<<' % ('', type(d2[key]), d2[key]))
    1451                                  different = True
    1452                                  continue
    1453                          if d1[key] == d2[key]:
    1454                                  _log.info('%25.25s:  both = >>>%s<<<' % (key, d1[key]))
    1455                          else:
    1456                                  _log.info('%25.25s: dict1 = >>>%s<<<' % (key, d1[key]))
    1457                                  _log.info('%25.25s  dict2 = >>>%s<<<' % ('', d2[key]))
    1458                                  different = True
    1459                  else:
    1460                          _log.info('%25.25s: %50.50s | <MISSING>' % (key, '>>>%s<<<' % d1[key]))
    1461                          different = True
    1462          for key in keys_d2:
    1463                  if key in keys_d1:
    1464                          continue
    1465                  _log.info('%25.25s: %50.50s | %.50s' % (key, '<MISSING>', '>>>%s<<<' % d2[key]))
    1466                  different = True
    1467          if different:
    1468                  _log.info('dict-likes appear to be different from each other')
    1469                  return False
    1470          _log.info('dict-likes appear equal to each other')
    1471          return True
    1472
    1473  #---------------------------------------------------------------------------
    1474 -def format_dict_likes_comparison(d1, d2, title_left=None, title_right=None, left_margin=0, key_delim=' || ', data_delim=' | ', missing_string='=/=', difference_indicator='! ', ignore_diff_in_keys=None):
    1475
    1476          _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title_left, '', '"%s" '), type(d1), coalesce(title_right, '', '"%s" '), type(d2))
    1477          append_type = False
    1478          if None not in [title_left, title_right]:
    1479                  append_type = True
    1480                  type_left = type(d1)
    1481                  type_right = type(d2)
    1482          if title_left is None:
    1483                  title_left = '%s' % type_left
    1484          if title_right is None:
    1485                  title_right = '%s' % type_right
    1486
    1487          try: d1 = dict(d1)
    1488          except TypeError: pass
    1489          try: d2 = dict(d2)
    1490          except TypeError: pass
    1491          keys_d1 = d1.keys()
    1492          keys_d2 = d2.keys()
    1493          data = {}
    1494          for key in keys_d1:
    1495                  data[key] = [d1[key], ' ']
    1496                  if key in d2:
    1497                          data[key][1] = d2[key]
    1498          for key in keys_d2:
    1499                  if key in keys_d1:
    1500                          continue
    1501                  data[key] = [' ', d2[key]]
    1502          max1 = max([ len('%s' % k) for k in keys_d1 ])
    1503          max2 = max([ len('%s' % k) for k in keys_d2 ])
    1504          max_len = max(max1, max2, len(_('<type>')))
    1505          max_key_len_str = '%' + '%s.%s' % (max_len, max_len) + 's'
    1506          max1 = max([ len('%s' % d1[k]) for k in keys_d1 ])
    1507          max2 = max([ len('%s' % d2[k]) for k in keys_d2 ])
    1508          max_data_len = min(max(max1, max2), 100)
    1509          max_data_len_str = '%' + '%s.%s' % (max_data_len, max_data_len) + 's'
    1510          diff_indicator_len_str = '%' + '%s.%s' % (len(difference_indicator), len(difference_indicator)) + 's'
    1511          line_template = (' ' * left_margin) + diff_indicator_len_str + max_key_len_str + key_delim + max_data_len_str + data_delim + '%s'
    1512
    1513          lines = []
    1514          # debugging:
    1515          #lines.append(u'                                        (40 regular spaces)')
    1516          #lines.append((u' ' * 40) + u"(u' ' * 40)")
    1517          #lines.append((u'%40.40s' % u'') + u"(u'%40.40s' % u'')")
    1518          #lines.append((u'%40.40s' % u' ') + u"(u'%40.40s' % u' ')")
    1519          #lines.append((u'%40.40s' % u'.') + u"(u'%40.40s' % u'.')")
    1520          #lines.append(line_template)
    1521          lines.append(line_template % ('', '', title_left, title_right))
    1522          if append_type:
    1523                  lines.append(line_template % ('', _('<type>'), type_left, type_right))
    1524
    1525          if ignore_diff_in_keys is None:
    1526                  ignore_diff_in_keys = []
    1527
    1528          for key in keys_d1:
    1529                  append_type = False
    1530                  txt_left_col = '%s' % d1[key]
    1531                  try:
    1532                          txt_right_col = '%s' % d2[key]
    1533                          if type(d1[key]) != type(d2[key]):
    1534                                  append_type = True
    1535                  except KeyError:
    1536                          txt_right_col = missing_string
    1537                  lines.append(line_template % (
    1538                          bool2subst (
    1539                                  ((txt_left_col == txt_right_col) or (key in ignore_diff_in_keys)),
    1540                                  '',
    1541                                  difference_indicator
    1542                          ),
    1543                          key,
    1544                          shorten_text(txt_left_col, max_data_len),
    1545                          shorten_text(txt_right_col, max_data_len)
    1546                  ))
    1547                  if append_type:
    1548                          lines.append(line_template % (
    1549                                  '',
    1550                                  _('<type>'),
    1551                                  shorten_text('%s' % type(d1[key]), max_data_len),
    1552                                  shorten_text('%s' % type(d2[key]), max_data_len)
    1553                          ))
    1554
    1555          for key in keys_d2:
    1556                  if key in keys_d1:
    1557                          continue
    1558                  lines.append(line_template % (
    1559                          bool2subst((key in ignore_diff_in_keys), '', difference_indicator),
    1560                          key,
    1561                          shorten_text(missing_string, max_data_len),
    1562                          shorten_text('%s' % d2[key], max_data_len)
    1563                  ))
    1564
    1565          return lines
    1566
    1567  #---------------------------------------------------------------------------
    1568 -def format_dict_like(d, relevant_keys=None, template=None, missing_key_template='<[%(key)s] MISSING>', left_margin=0, tabular=False, value_delimiters=('>>>', '<<<'), eol='\n', values2ignore=None):
    1569          if values2ignore is None:
    1570                  values2ignore = []
    1571          if template is not None:
    1572                  # all keys in template better exist in d
    1573                  try:
    1574                          return template % d
    1575                  except KeyError:
    1576                          # or else
    1577                          _log.exception('template contains %%()s key(s) which do not exist in data dict')
    1578                  # try to extend dict <d> to contain all required keys,
    1579                  # for that to work <relevant_keys> better list all
    1580                  # keys used in <template>
    1581                  if relevant_keys is not None:
    1582                          for key in relevant_keys:
    1583                                  try:
    1584                                          d[key]
    1585                                  except KeyError:
    1586                                          d[key] = missing_key_template % {'key': key}
    1587                          return template % d
    1588
    1589          if relevant_keys is None:
    1590                  relevant_keys = list(d.keys())
    1591          lines = []
    1592          if value_delimiters is None:
    1593                  delim_left = ''
    1594                  delim_right = ''
    1595          else:
    1596                  delim_left, delim_right = value_delimiters
    1597          if tabular:
    1598                  max_len = max([ len('%s' % k) for k in relevant_keys ])
    1599                  max_len_str = '%s.%s' % (max_len, max_len)
    1600                  line_template = (' ' * left_margin) + '%' + max_len_str + ('s: %s%%s%s' % (delim_left, delim_right))
    1601          else:
    1602                  line_template = (' ' * left_margin) + '%%s: %s%%s%s' % (delim_left, delim_right)
    1603          for key in relevant_keys:
    1604                  try:
    1605                          val = d[key]
    1606                  except KeyError:
    1607                          continue
    1608                  if val not in values2ignore:
    1609                          lines.append(line_template % (key, val))
    1610          if eol is None:
    1611                  return lines
    1612          return eol.join(lines)
    1613
    1614  #---------------------------------------------------------------------------
    1615 -def dicts2table(dict_list, left_margin=0, eol='\n', keys2ignore=None, column_labels=None, show_only_changes=False, equality_value='<=>', date_format=None):     #, relevant_keys=None, template=None
    1616          """Each dict in <dict_list> becomes a column.
    1617
    1618          - each key of dict becomes a row label, unless in keys2ignore
    1619
    1620          - each entry in the <column_labels> list becomes a column title
    1621          """
    1622          keys2show = []
    1623          col_max_width = {}
    1624          max_width_of_row_label_col = 0
    1625          col_label_key = '__________#header#__________'
    1626          if keys2ignore is None:
    1627                  keys2ignore = []
    1628          if column_labels is not None:
    1629                  keys2ignore.append(col_label_key)
    1630
    1631          # extract keys from all dicts and calculate column sizes
    1632          for dict_idx in range(len(dict_list)):
    1633                  # convert potentially dict-*like* into dict
    1634                  d = dict(dict_list[dict_idx])
    1635                  # add max-len column label row from <column_labels> list, if available
    1636                  if column_labels is not None:
    1637                          d[col_label_key] = max(column_labels[dict_idx].split('\n'), key = len)
    1638                  field_lengths = []
    1639                  # loop over all keys in this dict
    1640                  for key in d.keys():
    1641                          # ignore this key
    1642                          if key in keys2ignore:
    1643                                  continue
    1644                          # remember length of value when displayed
    1645                          if isinstance(d[key], pydt.datetime):
    1646                                  if date_format is None:
    1647                                          field_lengths.append(len('%s' % d[key]))
    1648                                  else:
    1649                                          field_lengths.append(len(d[key].strftime(date_format)))
    1650                          else:
    1651                                  field_lengths.append(len('%s' % d[key]))
    1652                          if key in keys2show:
    1653                                  continue
    1654                          keys2show.append(key)
    1655                          max_width_of_row_label_col = max(max_width_of_row_label_col, len('%s' % key))
    1656                  col_max_width[dict_idx] = max(field_lengths)
    1657
    1658          # pivot data into dict of lists per line
    1659          lines = { k: [] for k in keys2show }
    1660          prev_vals = {}
    1661          for dict_idx in range(len(dict_list)):
    1662                  max_width_this_col = max(col_max_width[dict_idx], len(equality_value)) if show_only_changes else col_max_width[dict_idx]
    1663                  max_len_str = '%s.%s' % (max_width_this_col, max_width_this_col)
    1664                  field_template = ' %' + max_len_str + 's'
    1665                  d = dict_list[dict_idx]
    1666                  for key in keys2show:
    1667                          try:
    1668                                  val = d[key]
    1669                          except KeyError:
    1670                                  lines[key].append(field_template % _('<missing>'))
    1671                                  continue
    1672                          if isinstance(val, pydt.datetime):
    1673                                  if date_format is not None:
    1674                                          val = val.strftime(date_format)
    1675                          lines[key].append(field_template % val)
    1676                          if show_only_changes:
    1677                                  if key not in prev_vals:
    1678                                          prev_vals[key] = '%s' % lines[key][-1]
    1679                                          continue
    1680                                  if lines[key][-1] != prev_vals[key]:
    1681                                          prev_vals[key] = '%s' % lines[key][-1]
    1682                                          continue
    1683                                  lines[key][-1] = field_template % equality_value
    1684
    1685          # format data into table
    1686          table_lines = []
    1687          max_len_str = '%s.%s' % (max_width_of_row_label_col, max_width_of_row_label_col)
    1688          row_label_template = '%' + max_len_str + 's'
    1689          for key in lines:
    1690                  # row label (= key) into first column
    1691                  line = (' ' * left_margin) + row_label_template % key + '|'
    1692                  # append list values as subsequent columns
    1693                  line += '|'.join(lines[key])
    1694                  table_lines.append(line)
    1695
    1696          # insert lines with column labels (column headers) if any
    1697          if column_labels is not None:
    1698                  # first column contains row labels, so no column label needed
    1699                  table_header_line_w_col_labels = (' ' * left_margin) + row_label_template % ''
    1700                  # second table header line: horizontal separator
    1701                  table_header_line_w_separator = (' ' * left_margin) + u_box_horiz_single * (max_width_of_row_label_col)
    1702                  max_col_label_widths = [ max(col_max_width[dict_idx], len(equality_value)) for dict_idx in range(len(dict_list)) ]
    1703                  for col_idx in range(len(column_labels)):
    1704                          max_len_str = '%s.%s' % (max_col_label_widths[col_idx], max_col_label_widths[col_idx])
    1705                          col_label_template = '%' + max_len_str + 's'
    1706                          table_header_line_w_col_labels += '| '
    1707                          table_header_line_w_col_labels += col_label_template % column_labels[col_idx]
    1708                          table_header_line_w_separator += '%s%s' % (u_box_plus, u_box_horiz_single)
    1709                          table_header_line_w_separator += u_box_horiz_single * max_col_label_widths[col_idx]
    1710                  table_lines.insert(0, table_header_line_w_separator)
    1711                  table_lines.insert(0, table_header_line_w_col_labels)
    1712
    1713          if eol is None:
    1714                  return table_lines
    1715
    1716          return ('|' + eol).join(table_lines) + '|' + eol
    1717
    1718  #---------------------------------------------------------------------------
    1719 -def normalize_dict_like(d, required_keys, missing_key_template='<[%(key)s] MISSING>'):
    1720          for key in required_keys:
    1721                  try:
    1722                          d[key]
    1723                  except KeyError:
    1724                          if missing_key_template is None:
    1725                                  d[key] = None
    1726                          else:
    1727                                  d[key] = missing_key_template % {'key': key}
    1728          return d
    1729
    1730  #---------------------------------------------------------------------------
    1731 -def enumerate_removable_partitions():
    1732          try:
    1733                  import pyudev
    1734                  import psutil
    1735          except ImportError:
    1736                  _log.error('pyudev and/or psutil not installed')
    1737                  return {}
    1738
    1739          removable_partitions = {}
    1740          ctxt = pyudev.Context()
    1741          removable_devices = [ dev for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk') if dev.attributes.get('removable') == b'1' ]
    1742          all_mounted_partitions = { part.device: part for part in psutil.disk_partitions() }
    1743          for device in removable_devices:
    1744                  _log.debug('removable device: %s', device.properties['ID_MODEL'])
    1745                  partitions_on_removable_device = {
    1746                          part.device_node: {
    1747                                  'type': device.properties['ID_TYPE'],
    1748                                  'bus': device.properties['ID_BUS'],
    1749                                  'device': device.properties['DEVNAME'],
    1750                                  'partition': part.properties['DEVNAME'],
    1751                                  'vendor': part.properties['ID_VENDOR'],
    1752                                  'model': part.properties['ID_MODEL'],
    1753                                  'fs_label': part.properties['ID_FS_LABEL'],
    1754                                  'is_mounted': False,
    1755                                  'mountpoint': None,
    1756                                  'fs_type': None,
    1757                                  'size_in_bytes': -1,
    1758                                  'bytes_free': 0
    1759                          } for part in ctxt.list_devices(subsystem='block', DEVTYPE='partition', parent=device)
    1760                  }
    1761                  for part in partitions_on_removable_device:
    1762                          try:
    1763                                  partitions_on_removable_device[part]['mountpoint'] = all_mounted_partitions[part].mountpoint
    1764                                  partitions_on_removable_device[part]['is_mounted'] = True
    1765                                  partitions_on_removable_device[part]['fs_type'] = all_mounted_partitions[part].fstype
    1766                                  du = shutil.disk_usage(all_mounted_partitions[part].mountpoint)
    1767                                  partitions_on_removable_device[part]['size_in_bytes'] = du.total
    1768                                  partitions_on_removable_device[part]['bytes_free'] = du.free
    1769                          except KeyError:
    1770                                  pass                    # not mounted
    1771                  removable_partitions.update(partitions_on_removable_device)
    1772          return removable_partitions
    1773
    1774          # debugging:
    1775          #ctxt = pyudev.Context()
    1776          #for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk'):# if dev.attributes.get('removable') == b'1':
    1777          #       for a in dev.attributes.available_attributes:
    1778          #               print(a, dev.attributes.get(a))
    1779          #       for key, value in dev.items():
    1780          #               print('{key}={value}'.format(key=key, value=value))
    1781          #       print('---------------------------')
    1782
    1783  #---------------------------------------------------------------------------
    1784 -def enumerate_optical_writers():
    1785          try:
    1786                  import pyudev
    1787          except ImportError:
    1788                  _log.error('pyudev not installed')
    1789                  return []
    1790
    1791          optical_writers = []
    1792          ctxt = pyudev.Context()
    1793          for dev in [ dev for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk') if dev.properties.get('ID_CDROM_CD_RW', None) == '1' ]:
    1794                  optical_writers.append ({
    1795                          'type': dev.properties['ID_TYPE'],
    1796                          'bus': dev.properties['ID_BUS'],
    1797                          'device': dev.properties['DEVNAME'],
    1798                          'model': dev.properties['ID_MODEL']
    1799                  })
    1800          return optical_writers
    1801
    1802  #---------------------------------------------------------------------------
    1803  #---------------------------------------------------------------------------
    1804 -def prompted_input(prompt=None, default=None):
    1805          """Obtains entry from standard input.
    1806
    1807          prompt: Prompt text to display in standard output
    1808          default: Default value (for user to press enter only)
    1809          CTRL-C: aborts and returns None
    1810          """
    1811          if prompt is None:
    1812                  msg = '(CTRL-C aborts)'
    1813          else:
    1814                  msg = '%s (CTRL-C aborts)' % prompt
    1815
    1816          if default is None:
    1817                  msg = msg + ': '
    1818          else:
    1819                  msg = '%s [%s]: ' % (msg, default)
    1820
    1821          try:
    1822                  usr_input = input(msg)
    1823          except KeyboardInterrupt:
    1824                  return None
    1825
    1826          if usr_input == '':
    1827                  return default
    1828
    1829          return usr_input
    1830
    1831  #===========================================================================
    1832  # image handling tools
    1833  #---------------------------------------------------------------------------
    1834  # builtin (ugly but tried and true) fallback icon
    1835  __icon_serpent = \
    1836  """x\xdae\x8f\xb1\x0e\x83 \x10\x86w\x9f\xe2\x92\x1blb\xf2\x07\x96\xeaH:0\xd6\
    1837  \xc1\x85\xd5\x98N5\xa5\xef?\xf5N\xd0\x8a\xdcA\xc2\xf7qw\x84\xdb\xfa\xb5\xcd\
    1838  \xd4\xda;\xc9\x1a\xc8\xb6\xcd<\xb5\xa0\x85\x1e\xeb\xbc\xbc7b!\xf6\xdeHl\x1c\
    1839  \x94\x073\xec<*\xf7\xbe\xf7\x99\x9d\xb21~\xe7.\xf5\x1f\x1c\xd3\xbdVlL\xc2\
    1840  \xcf\xf8ye\xd0\x00\x90\x0etH \x84\x80B\xaa\x8a\x88\x85\xc4(U\x9d$\xfeR;\xc5J\
    1841  \xa6\x01\xbbt9\xceR\xc8\x81e_$\x98\xb9\x9c\xa9\x8d,y\xa9t\xc8\xcf\x152\xe0x\
    1842  \xe9$\xf5\x07\x95\x0cD\x95t:\xb1\x92\xae\x9cI\xa8~\x84\x1f\xe0\xa3ec"""
    1843
    1844 -def get_icon(wx=None):
    1845
    1846          paths = gmPaths(app_name = 'gnumed', wx = wx)
    1847
    1848          candidates = [
    1849                  os.path.join(paths.system_app_data_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'),
    1850                  os.path.join(paths.local_base_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'),
    1851                  os.path.join(paths.system_app_data_dir, 'bitmaps', 'serpent.png'),
    1852                  os.path.join(paths.local_base_dir, 'bitmaps', 'serpent.png')
    1853          ]
    1854
    1855          found_as = None
    1856          for candidate in candidates:
    1857                  try:
    1858                          open(candidate, 'r').close()
    1859                          found_as = candidate
    1860                          break
    1861                  except IOError:
    1862                          _log.debug('icon not found in [%s]', candidate)
    1863
    1864          if found_as is None:
    1865                  _log.warning('no icon file found, falling back to builtin (ugly) icon')
    1866                  icon_bmp_data = wx.BitmapFromXPMData(pickle.loads(zlib.decompress(__icon_serpent)))
    1867                  icon.CopyFromBitmap(icon_bmp_data)
    1868          else:
    1869                  _log.debug('icon found in [%s]', found_as)
    1870                  icon = wx.Icon()
    1871                  try:
    1872                          icon.LoadFile(found_as, wx.BITMAP_TYPE_ANY)             #_PNG
    1873                  except AttributeError:
    1874                          _log.exception("this platform doesn't support wx.Icon().LoadFile()")
    1875
    1876          return icon
    1877
    1878  #---------------------------------------------------------------------------
    1879 -def create_qrcode(text=None, filename=None, qr_filename=None, verbose=False):
    1880          assert (not ((text is None) and (filename is None))), 'either <text> or <filename> must be specified'
    1881
    1882          try:
    1883                  import pyqrcode
    1884          except ImportError:
    1885                  _log.exception('cannot import <pyqrcode>')
    1886                  return None
    1887          if text is None:
    1888                  with io.open(filename, mode = 'rt', encoding = 'utf8') as input_file:
    1889                          text = input_file.read()
    1890          if qr_filename is None:
    1891                  if filename is None:
    1892                          qr_filename = get_unique_filename(prefix = 'gm-qr-', suffix = '.png')
    1893                  else:
    1894                          qr_filename = get_unique_filename (
    1895                                  prefix = fname_stem(filename) + '-',
    1896                                  suffix = fname_extension(filename) + '.png'
    1897                          )
    1898          _log.debug('[%s] -> [%s]', filename, qr_filename)
    1899          qr = pyqrcode.create(text, encoding = 'utf8')
    1900          if verbose:
    1901                  print('input file:', filename)
    1902                  print('output file:', qr_filename)
    1903                  print('text to encode:', text)
    1904                  print(qr.terminal())
    1905          qr.png(qr_filename, quiet_zone = 1)
    1906          return qr_filename
    1907
    1908  #===========================================================================
    1909  # main
    1910  #---------------------------------------------------------------------------
    1911  if __name__ == '__main__':
    1912
    1913          if len(sys.argv) < 2:
    1914                  sys.exit()
    1915
    1916          if sys.argv[1] != 'test':
    1917                  sys.exit()
    1918
    1919          # for testing:
    1920          logging.basicConfig(level = logging.DEBUG)
    1921          from Gnumed.pycommon import gmI18N
    1922          gmI18N.activate_locale()
    1923          gmI18N.install_domain()
    1924
    1925          #-----------------------------------------------------------------------
    1926 -        def test_input2decimal():
    1927
    1928                  tests = [
    1929                          [None, False],
    1930
    1931                          ['', False],
    1932                          [' 0 ', True, 0],
    1933
    1934                          [0, True, 0],
    1935                          [0.0, True, 0],
    1936                          [.0, True, 0],
    1937                          ['0', True, 0],
    1938                          ['0.0', True, 0],
    1939                          ['0,0', True, 0],
    1940                          ['00.0', True, 0],
    1941                          ['.0', True, 0],
    1942                          [',0', True, 0],
    1943
    1944                          [0.1, True, decimal.Decimal('0.1')],
    1945                          [.01, True, decimal.Decimal('0.01')],
    1946                          ['0.1', True, decimal.Decimal('0.1')],
    1947                          ['0,1', True, decimal.Decimal('0.1')],
    1948                          ['00.1', True, decimal.Decimal('0.1')],
    1949                          ['.1', True, decimal.Decimal('0.1')],
    1950                          [',1', True, decimal.Decimal('0.1')],
    1951
    1952                          [1, True, 1],
    1953                          [1.0, True, 1],
    1954                          ['1', True, 1],
    1955                          ['1.', True, 1],
    1956                          ['1,', True, 1],
    1957                          ['1.0', True, 1],
    1958                          ['1,0', True, 1],
    1959                          ['01.0', True, 1],
    1960                          ['01,0', True, 1],
    1961                          [' 01, ', True, 1],
    1962
    1963                          [decimal.Decimal('1.1'), True, decimal.Decimal('1.1')]
    1964                  ]
    1965                  for test in tests:
    1966                          conversion_worked, result = input2decimal(initial = test[0])
    1967
    1968                          expected2work = test[1]
    1969
    1970                          if conversion_worked:
    1971                                  if expected2work:
    1972                                          if result == test[2]:
    1973                                                  continue
    1974                                          else:
    1975                                                  print("ERROR (conversion result wrong): >%s<, expected >%s<, got >%s<" % (test[0], test[2], result))
    1976                                  else:
    1977                                          print("ERROR (conversion worked but was expected to fail): >%s<, got >%s<" % (test[0], result))
    1978                          else:
    1979                                  if not expected2work:
    1980                                          continue
    1981                                  else:
    1982                                          print("ERROR (conversion failed but was expected to work): >%s<, expected >%s<" % (test[0], test[2]))
    1983          #-----------------------------------------------------------------------
    1984 -        def test_input2int():
    1985                  print(input2int(0))
    1986                  print(input2int('0'))
    1987                  print(input2int('0', 0, 0))
    1988          #-----------------------------------------------------------------------
    1989 -        def test_coalesce():
    1990
    1991                  val = None
    1992                  print(val, coalesce(val, 'is None', 'is not None'))
    1993                  val = 1
    1994                  print(val, coalesce(val, 'is None', 'is not None'))
    1995                  return
    1996
    1997                  import datetime as dt
    1998                  print(coalesce(value2test = dt.datetime.now(), template4value = '-- %s --', function4value = ('strftime', '%Y-%m-%d')))
    1999
    2000                  print('testing coalesce()')
    2001                  print("------------------")
    2002                  tests = [
    2003                          [None, 'something other than <None>', None, None, 'something other than <None>'],
    2004                          ['Captain', 'Mr.', '%s.'[:4], 'Mr.', 'Capt.'],
    2005                          ['value to test', 'test 3 failed', 'template with "%s" included', None, 'template with "value to test" included'],
    2006                          ['value to test', 'test 4 failed', 'template with value not included', None, 'template with value not included'],
    2007                          [None, 'initial value was None', 'template4value: %s', None, 'initial value was None'],
    2008                          [None, 'initial value was None', 'template4value: %%(abc)s', None, 'initial value was None']
    2009                  ]
    2010                  passed = True
    2011                  for test in tests:
    2012                          result = coalesce (
    2013                                  value2test = test[0],
    2014                                  return_instead = test[1],
    2015                                  template4value = test[2],
    2016                                  template4instead = test[3]
    2017                          )
    2018                          if result != test[4]:
    2019                                  print("ERROR")
    2020                                  print("coalesce: (%s, %s, %s, %s)" % (test[0], test[1], test[2], test[3]))
    2021                                  print("expected:", test[4])
    2022                                  print("received:", result)
    2023                                  passed = False
    2024
    2025                  if passed:
    2026                          print("passed")
    2027                  else:
    2028                          print("failed")
    2029                  return passed
    2030          #-----------------------------------------------------------------------
    2031 -        def test_capitalize():
    2032                  print('testing capitalize() ...')
    2033                  success = True
    2034                  pairs = [
    2035                          # [original, expected result, CAPS mode]
    2036                          ['Boot', 'Boot', CAPS_FIRST_ONLY],
    2037                          ['boot', 'Boot', CAPS_FIRST_ONLY],
    2038                          ['booT', 'Boot', CAPS_FIRST_ONLY],
    2039                          ['BoOt', 'Boot', CAPS_FIRST_ONLY],
    2040                          ['boots-Schau', 'Boots-Schau', CAPS_WORDS],
    2041                          ['boots-sChau', 'Boots-Schau', CAPS_WORDS],
    2042                          ['boot camp', 'Boot Camp', CAPS_WORDS],
    2043                          ['fahrner-Kampe', 'Fahrner-Kampe', CAPS_NAMES],
    2044                          ['häkkönen', 'Häkkönen', CAPS_NAMES],
    2045                          ['McBurney', 'McBurney', CAPS_NAMES],
    2046                          ['mcBurney', 'McBurney', CAPS_NAMES],
    2047                          ['blumberg', 'Blumberg', CAPS_NAMES],
    2048                          ['roVsing', 'RoVsing', CAPS_NAMES],
    2049                          ['Özdemir', 'Özdemir', CAPS_NAMES],
    2050                          ['özdemir', 'Özdemir', CAPS_NAMES],
    2051                  ]
    2052                  for pair in pairs:
    2053                          result = capitalize(pair[0], pair[2])
    2054                          if result != pair[1]:
    2055                                  success = False
    2056                                  print('ERROR (caps mode %s): "%s" -> "%s", expected "%s"' % (pair[2], pair[0], result, pair[1]))
    2057
    2058                  if success:
    2059                          print("... SUCCESS")
    2060
    2061                  return success
    2062          #-----------------------------------------------------------------------
    2063 -        def test_import_module():
    2064                  print("testing import_module_from_directory()")
    2065                  path = sys.argv[1]
    2066                  name = sys.argv[2]
    2067                  try:
    2068                          mod = import_module_from_directory(module_path = path, module_name = name)
    2069                  except Exception:
    2070                          print("module import failed, see log")
    2071                          return False
    2072
    2073                  print("module import succeeded", mod)
    2074                  print(dir(mod))
    2075                  return True
    2076          #-----------------------------------------------------------------------
    2077 -        def test_mkdir():
    2078                  print("testing mkdir(%s)" % sys.argv[2])
    2079                  mkdir(sys.argv[2], 0o0700)
    2080          #-----------------------------------------------------------------------
    2081 -        def test_gmPaths():
    2082                  print("testing gmPaths()")
    2083                  print("-----------------")
    2084                  paths = gmPaths(wx=None, app_name='gnumed')
    2085                  print("user     config dir:", paths.user_config_dir)
    2086                  print("system   config dir:", paths.system_config_dir)
    2087                  print("local      base dir:", paths.local_base_dir)
    2088                  print("system app data dir:", paths.system_app_data_dir)
    2089                  print("working directory  :", paths.working_dir)
    2090                  print("temp directory     :", paths.tmp_dir)
    2091          #-----------------------------------------------------------------------
    2092 -        def test_none_if():
    2093                  print("testing none_if()")
    2094                  print("-----------------")
    2095                  tests = [
    2096                          [None, None, None],
    2097                          ['a', 'a', None],
    2098                          ['a', 'b', 'a'],
    2099                          ['a', None, 'a'],
    2100                          [None, 'a', None],
    2101                          [1, 1, None],
    2102                          [1, 2, 1],
    2103                          [1, None, 1],
    2104                          [None, 1, None]
    2105                  ]
    2106
    2107                  for test in tests:
    2108                          if none_if(value = test[0], none_equivalent = test[1]) != test[2]:
    2109                                  print('ERROR: none_if(%s) returned [%s], expected [%s]' % (test[0], none_if(test[0], test[1]), test[2]))
    2110
    2111                  return True
    2112          #-----------------------------------------------------------------------
    2113 -        def test_bool2str():
    2114                  tests = [
    2115                          [True, 'Yes', 'Yes', 'Yes'],
    2116                          [False, 'OK', 'not OK', 'not OK']
    2117                  ]
    2118                  for test in tests:
    2119                          if bool2str(test[0], test[1], test[2]) != test[3]:
    2120                                  print('ERROR: bool2str(%s, %s, %s) returned [%s], expected [%s]' % (test[0], test[1], test[2], bool2str(test[0], test[1], test[2]), test[3]))
    2121
    2122                  return True
    2123          #-----------------------------------------------------------------------
    2124 -        def test_bool2subst():
    2125
    2126                  print(bool2subst(True, 'True', 'False', 'is None'))
    2127                  print(bool2subst(False, 'True', 'False', 'is None'))
    2128                  print(bool2subst(None, 'True', 'False', 'is None'))
    2129          #-----------------------------------------------------------------------
    2130 -        def test_get_unique_filename():
    2131                  print(get_unique_filename())
    2132                  print(get_unique_filename(prefix='test-'))
    2133                  print(get_unique_filename(suffix='tst'))
    2134                  print(get_unique_filename(prefix='test-', suffix='tst'))
    2135                  print(get_unique_filename(tmp_dir='/home/ncq/Archiv/'))
    2136          #-----------------------------------------------------------------------
    2137 -        def test_size2str():
    2138                  print("testing size2str()")
    2139                  print("------------------")
    2140                  tests = [0, 1, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, 100000000000, 1000000000000, 10000000000000]
    2141                  for test in tests:
    2142                          print(size2str(test))
    2143          #-----------------------------------------------------------------------
    2144 -        def test_unwrap():
    2145
    2146                  test = """
    2147  second line\n
    2148          3rd starts with tab  \n
    2149   4th with a space       \n
    2150
    2151  6th
    2152
    2153  """
    2154                  print(unwrap(text = test, max_length = 25))
    2155          #-----------------------------------------------------------------------
    2156 -        def test_wrap():
    2157                  test = 'line 1\nline 2\nline 3'
    2158
    2159                  print("wrap 5-6-7 initial 0, subsequent 0")
    2160                  print(wrap(test, 5))
    2161                  print()
    2162                  print(wrap(test, 6))
    2163                  print()
    2164                  print(wrap(test, 7))
    2165                  print("-------")
    2166                  input()
    2167                  print("wrap 5 initial 1-1-3, subsequent 1-3-1")
    2168                  print(wrap(test, 5, ' ', ' '))
    2169                  print()
    2170                  print(wrap(test, 5, ' ', '   '))
    2171                  print()
    2172                  print(wrap(test, 5, '   ', ' '))
    2173                  print("-------")
    2174                  input()
    2175                  print("wrap 6 initial 1-1-3, subsequent 1-3-1")
    2176                  print(wrap(test, 6, ' ', ' '))
    2177                  print()
    2178                  print(wrap(test, 6, ' ', '   '))
    2179                  print()
    2180                  print(wrap(test, 6, '   ', ' '))
    2181                  print("-------")
    2182                  input()
    2183                  print("wrap 7 initial 1-1-3, subsequent 1-3-1")
    2184                  print(wrap(test, 7, ' ', ' '))
    2185                  print()
    2186                  print(wrap(test, 7, ' ', '   '))
    2187                  print()
    2188                  print(wrap(test, 7, '   ', ' '))
    2189          #-----------------------------------------------------------------------
    2190 -        def test_md5():
    2191                  print('md5 %s: %s' % (sys.argv[2], file2md5(sys.argv[2])))
    2192                  print('chunked md5 %s: %s' % (sys.argv[2], file2chunked_md5(sys.argv[2])))
    2193          #-----------------------------------------------------------------------
    2194 -        def test_unicode():
    2195                  print(u_link_symbol * 10)
    2196          #-----------------------------------------------------------------------
    2197 -        def test_xml_escape():
    2198                  print(xml_escape_string('<'))
    2199                  print(xml_escape_string('>'))
    2200                  print(xml_escape_string('&'))
    2201          #-----------------------------------------------------------------------
    2202 -        def test_tex_escape():
    2203                  tests = ['\\', '^', '~', '{', '}', '%',  '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234']
    2204                  tests.append('  '.join(tests))
    2205                  for test in tests:
    2206                          print('%s:' % test, tex_escape_string(test))
    2207
    2208          #-----------------------------------------------------------------------
    2209 -        def test_rst2latex_snippet():
    2210                  tests = ['\\', '^', '~', '{', '}', '%',  '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234']
    2211                  tests.append('  '.join(tests))
    2212                  tests.append('C:\Windows\Programme\System 32\lala.txt')
    2213                  tests.extend([
    2214                          'should be identical',
    2215                          'text *some text* text',
    2216                          """A List
    2217  ======
    2218
    2219  1. 1
    2220  2. 2
    2221
    2222  3. ist-list
    2223  1. more
    2224  2. noch was ü
    2225  #. nummer x"""
    2226                  ])
    2227                  for test in tests:
    2228                          print('==================================================')
    2229                          print('raw:')
    2230                          print(test)
    2231                          print('---------')
    2232                          print('ReST 2 LaTeX:')
    2233                          latex = rst2latex_snippet(test)
    2234                          print(latex)
    2235                          if latex.strip() == test.strip():
    2236                                  print('=> identical')
    2237                          print('---------')
    2238                          print('tex_escape_string:')
    2239                          print(tex_escape_string(test))
    2240                          input()
    2241
    2242          #-----------------------------------------------------------------------
    2243 -        def test_strip_trailing_empty_lines():
    2244                  tests = [
    2245                          'one line, no embedded line breaks  ',
    2246                          'one line\nwith embedded\nline\nbreaks\n   '
    2247                  ]
    2248                  for test in tests:
    2249                          print('as list:')
    2250                          print(strip_trailing_empty_lines(text = test, eol='\n', return_list = True))
    2251                          print('as string:')
    2252                          print('>>>%s<<<' % strip_trailing_empty_lines(text = test, eol='\n', return_list = False))
    2253                  tests = [
    2254                          ['list', 'without', 'empty', 'trailing', 'lines'],
    2255                          ['list', 'with', 'empty', 'trailing', 'lines', '', '  ', '']
    2256                  ]
    2257                  for test in tests:
    2258                          print('as list:')
    2259                          print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = True))
    2260                          print('as string:')
    2261                          print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = False))
    2262          #-----------------------------------------------------------------------
    2263 -        def test_fname_stem():
    2264                  tests = [
    2265                          r'abc.exe',
    2266                          r'\abc.exe',
    2267                          r'c:\abc.exe',
    2268                          r'c:\d\abc.exe',
    2269                          r'/home/ncq/tmp.txt',
    2270                          r'~/tmp.txt',
    2271                          r'./tmp.txt',
    2272                          r'./.././tmp.txt',
    2273                          r'tmp.txt'
    2274                  ]
    2275                  for t in tests:
    2276                          print("[%s] -> [%s]" % (t, fname_stem(t)))
    2277          #-----------------------------------------------------------------------
    2278 -        def test_dir_is_empty():
    2279                  print(sys.argv[2], 'empty:', dir_is_empty(sys.argv[2]))
    2280
    2281          #-----------------------------------------------------------------------
    2282 -        def test_compare_dicts():
    2283                  d1 = {}
    2284                  d2 = {}
    2285                  d1[1] = 1
    2286                  d1[2] = 2
    2287                  d1[3] = 3
    2288                  # 4
    2289                  d1[5] = 5
    2290
    2291                  d2[1] = 1
    2292                  d2[2] = None
    2293                  # 3
    2294                  d2[4] = 4
    2295
    2296                  #compare_dict_likes(d1, d2)
    2297
    2298                  d1 = {1: 1, 2: 2}
    2299                  d2 = {1: 1, 2: 2}
    2300
    2301                  #compare_dict_likes(d1, d2, 'same1', 'same2')
    2302                  print(format_dict_like(d1, tabular = False))
    2303                  print(format_dict_like(d1, tabular = True))
    2304                  #print(format_dict_like(d2))
    2305
    2306          #-----------------------------------------------------------------------
    2307 -        def test_format_compare_dicts():
    2308                  d1 = {}
    2309                  d2 = {}
    2310                  d1[1] = 1
    2311                  d1[2] = 2
    2312                  d1[3] = 3
    2313                  # 4
    2314                  d1[5] = 5
    2315
    2316                  d2[1] = 1
    2317                  d2[2] = None
    2318                  # 3
    2319                  d2[4] = 4
    2320
    2321                  print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2')))
    2322
    2323                  d1 = {1: 1, 2: 2}
    2324                  d2 = {1: 1, 2: 2}
    2325
    2326                  print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2')))
    2327
    2328          #-----------------------------------------------------------------------
    2329 -        def test_rm_dir():
    2330                  rmdir('cx:\windows\system3__2xxxxxxxxxxxxx')
    2331
    2332          #-----------------------------------------------------------------------
    2333 -        def test_rm_dir_content():
    2334                  #print(rm_dir_content('cx:\windows\system3__2xxxxxxxxxxxxx'))
    2335                  print(rm_dir_content('/tmp/user/1000/tmp'))
    2336
    2337          #-----------------------------------------------------------------------
    2338 -        def test_strip_prefix():
    2339                  tests = [
    2340                          ('', '', ''),
    2341                          ('a', 'a', ''),
    2342                          ('GMd: a window title', _GM_TITLE_PREFIX + ':', 'a window title'),
    2343                          ('\.br\MICROCYTES+1\.br\SPHEROCYTES       present\.br\POLYCHROMASIAmoderate\.br\\', '\.br\\', 'MICROCYTES+1\.br\SPHEROCYTES       present\.br\POLYCHROMASIAmoderate\.br\\')
    2344                  ]
    2345                  for test in tests:
    2346                          text, prefix, expect = test
    2347                          result = strip_prefix(text, prefix, remove_whitespace = True)
    2348                          if result == expect:
    2349                                  continue
    2350                          print('test failed:', test)
    2351                          print('result:', result)
    2352
    2353          #-----------------------------------------------------------------------
    2354 -        def test_shorten_text():
    2355                  tst = [
    2356                          ('123', 1),
    2357                          ('123', 2),
    2358                          ('123', 3),
    2359                          ('123', 4),
    2360                          ('', 1),
    2361                          ('1', 1),
    2362                          ('12', 1),
    2363                          ('', 2),
    2364                          ('1', 2),
    2365                          ('12', 2),
    2366                          ('123', 2)
    2367                  ]
    2368                  for txt, lng in tst:
    2369                          print('max', lng, 'of', txt, '=', shorten_text(txt, lng))
    2370          #-----------------------------------------------------------------------
    2371 -        def test_fname_sanitize():
    2372                  tests = [
    2373                          '/tmp/test.txt',
    2374                          '/tmp/ test.txt',
    2375                          '/tmp/ tes\\t.txt',
    2376                          'test'
    2377                  ]
    2378                  for test in tests:
    2379                          print (test, fname_sanitize(test))
    2380
    2381          #-----------------------------------------------------------------------
    2382 -        def test_create_qrcode():
    2383                  print(create_qrcode(text = sys.argv[2], filename=None, qr_filename=None, verbose = True))
    2384
    2385          #-----------------------------------------------------------------------
    2386 -        def test_enumerate_removable_partitions():
    2387                  parts = enumerate_removable_partitions()
    2388                  for part_name in parts:
    2389                          part = parts[part_name]
    2390                          print(part['device'])
    2391                          print(part['partition'])
    2392                          if part['is_mounted']:
    2393                                  print('%s@%s: %s on %s by %s @ %s (FS=%s: %s free of %s total)' % (
    2394                                          part['type'],
    2395                                          part['bus'],
    2396                                          part['fs_label'],
    2397                                          part['model'],
    2398                                          part['vendor'],
    2399                                          part['mountpoint'],
    2400                                          part['fs_type'],
    2401                                          part['bytes_free'],
    2402                                          part['size_in_bytes']
    2403                                  ))
    2404                          else:
    2405                                  print('%s@%s: %s on %s by %s (not mounted)' % (
    2406                                          part['type'],
    2407                                          part['bus'],
    2408                                          part['fs_label'],
    2409                                          part['model'],
    2410                                          part['vendor']
    2411                                  ))
    2412
    2413          #-----------------------------------------------------------------------
    2414 -        def test_enumerate_optical_writers():
    2415                  for writer in enumerate_optical_writers():
    2416                          print('%s@%s: %s @ %s' % (
    2417                                  writer['type'],
    2418                                  writer['bus'],
    2419                                  writer['model'],
    2420                                  writer['device']
    2421                          ))
    2422
    2423          #-----------------------------------------------------------------------
    2424 -        def test_copy_tree_content():
    2425                  print(sys.argv[2], '->', sys.argv[3])
    2426                  print(copy_tree_content(sys.argv[2], sys.argv[3]))
    2427
    2428          #-----------------------------------------------------------------------
    2429 -        def test_mk_sandbox_dir():
    2430                  print(mk_sandbox_dir(base_dir = '/tmp/abcd/efg/h'))
    2431
    2432          #-----------------------------------------------------------------------
    2433 -        def test_make_table_from_dicts():
    2434                  dicts = [
    2435                          {'pkey': 1, 'value': 'a122'},
    2436                          {'pkey': 2, 'value': 'b23'},
    2437                          {'pkey': 3, 'value': 'c3'},
    2438                          {'pkey': 4, 'value': 'd4ssssssssssss'},
    2439                          {'pkey': 5, 'value': 'd4  asdfas '},
    2440                          {'pkey': 5, 'value': 'c5---'},
    2441                  ]
    2442                  with open('x.txt', 'w', encoding = 'utf8') as f:
    2443                          f.write(dicts2table(dicts, left_margin=2, eol='\n', keys2ignore=None, show_only_changes=True, column_labels = ['d1', 'd2', 'd3', 'd4', 'd5', 'd6']))
    2444                          #print(dicts2table(dicts, left_margin=2, eol='\n', keys2ignore=None, show_only_changes=True, column_labels = ['d1', 'd2', 'd3', 'd4', 'd5', 'd6']))
    2445
    2446          #-----------------------------------------------------------------------
    2447 -        def test_decorate_window_title():
    2448                  for txt in globals():
    2449                          print(decorate_window_title(txt))
    2450
    2451          #-----------------------------------------------------------------------
    2452          #test_coalesce()
    2453          #test_capitalize()
    2454          #test_import_module()
    2455          #test_mkdir()
    2456          #test_gmPaths()
    2457          #test_none_if()
    2458          #test_bool2str()
    2459          #test_bool2subst()
    2460          #test_get_unique_filename()
    2461          #test_size2str()
    2462          #test_wrap()
    2463          #test_input2decimal()
    2464          #test_input2int()
    2465          #test_unwrap()
    2466          #test_md5()
    2467          #test_unicode()
    2468          #test_xml_escape()
    2469          #test_strip_trailing_empty_lines()
    2470          #test_fname_stem()
    2471          #test_tex_escape()
    2472          #test_rst2latex_snippet()
    2473          #test_dir_is_empty()
    2474          #test_compare_dicts()
    2475          #test_rm_dir()
    2476          #test_rm_dir_content()
    2477          #test_strip_prefix()
    2478          #test_shorten_text()
    2479          #test_format_compare_dicts()
    2480          #test_fname_sanitize()
    2481          #test_create_qrcode()
    2482          #test_enumerate_removable_partitions()
    2483          #test_enumerate_optical_writers()
    2484          #test_copy_tree_content()
    2485          #test_mk_sandbox_dir()
    2486          test_make_table_from_dicts()
    2487          #test_decorate_window_title()
    2488
    2489  #===========================================================================
    2490