Gnumed.importers.gmDrugsFromCSV

Package Gnumed :: Package importers :: Module gmDrugsFromCSV |
[frames] | no frames]

# Source Code for Module Gnumed.importers.gmDrugsFromCSV

      1  # creates SQL to imports a list of mono-component drugs from a CSV file
      2
      3  import sys
      4  import io
      5
      6
      7  # GNUmed
      8  if __name__ == '__main__':
      9          sys.path.insert(0, '../../')
     10  from Gnumed.pycommon import gmLog2
     11  from Gnumed.pycommon import gmTools
     12
     13
     14  field_names = ['substance', 'product', 'form', 'company', 'strength_1', 'strength_2', 'strength_3', 'always_empty', 'unit']
     15  non_empty_fields = ['substance', 'product', 'form', 'company', 'strength_1', 'unit']
     16  numeric_fields = ['strength_1', 'strength_2', 'strength_3']
     17
     18
     19  SQL_start = """-- ---------------------------------------------------------
     20  -- data pack install script example
     21  --
     22  -- add a description here: Mono-substance drugs as available in India
     23  -- license: GPL v2 or later, manually transferred 3rd party data
     24  -- provided by Vaibhav Banait
     25  --
     26  -- http://wiki.gnumed.de/bin/view/Gnumed/GmManualReferenceData
     27  -- ---------------------------------------------------------
     28  -- enable this if running locally via
     29  -- psql -d gnumed_vXX -U gm-dbo -f install-data-pack.sql
     30  --SET default_transaction_read_only TO OFF;
     31
     32  -- ---------------------------------------------------------
     33  -- drop staging tables if needed
     34  \\unset ON_ERROR_STOP
     35  DROP TABLE staging.india_drugs CASCADE;
     36  \set ON_ERROR_STOP 1
     37
     38  -- ---------------------------------------------------------
     39  -- run everything else in one transaction
     40  BEGIN;
     41
     42  -- create staging tables as needed --
     43  CREATE TABLE staging.india_drugs (
     44          brand_name text,
     45          substance text,
     46          form text,
     47          strength numeric,
     48          unit text
     49  );
     50
     51  -- ---------------------------------------------------------
     52  -- insert data in staging table
     53  """
     54
     55
     56  SQL_stage_drug = """INSERT INTO staging.india_drugs (brand_name, substance, form, strength, unit) SELECT
     57          '%(brand_name)s',
     58          '%(substance)s',
     59          '%(form)s',
     60          gm.nullify_empty_string('%(strength)s')::numeric,
     61          '%(unit)s'
     62  WHERE NOT EXISTS (
     63          SELECT 1 FROM staging.india_drugs WHERE brand_name = '%(brand_name)s'
     64  );
     65  """
     66
     67
     68  SQL_end = """-- ---------------------------------------------------------
     69  -- transfer data to real tables
     70
     71  -- substances
     72  INSERT INTO ref.consumable_substance (description, amount, unit) SELECT
     73          DISTINCT ON (s_id.substance, s_id.strength, s_id.unit)
     74          s_id.substance, s_id.strength, s_id.unit
     75  FROM
     76          staging.india_drugs s_id
     77  WHERE
     78          s_id.strength IS NOT NULL
     79                  AND
     80          NOT EXISTS (
     81                  SELECT 1 FROM ref.consumable_substance r_cs WHERE
     82                          r_cs.description = s_id.substance
     83                                  AND
     84                          r_cs.amount = s_id.strength
     85                                  AND
     86                          r_cs.unit = s_id.unit
     87          )
     88  ;
     89
     90  -- drug products
     91  INSERT INTO ref.drug_product (description, preparation) SELECT
     92          s_id.brand_name, s_id.form
     93  FROM
     94          staging.india_drugs s_id
     95  WHERE NOT EXISTS (
     96          SELECT 1 FROM ref.drug_product r_bd WHERE
     97                  r_bd.description = s_id.brand_name
     98                          AND
     99                  r_bd.preparation = s_id.form
    100  );
    101
    102  -- link components
    103  INSERT INTO ref.lnk_substance2brand (fk_drug_product, fk_substance) SELECT
    104          (SELECT pk FROM ref.drug_product r_bd
    105           WHERE
    106                  r_bd.description = s_id.brand_name
    107                          AND
    108                  r_bd.preparation = s_id.form
    109          ),
    110          (SELECT pk FROM ref.consumable_substance r_cs WHERE
    111                  r_cs.description = s_id.substance
    112                          AND
    113                  r_cs.amount = s_id.strength
    114                          AND
    115                  r_cs.unit = s_id.unit
    116          )
    117  FROM
    118          staging.india_drugs s_id
    119  WHERE NOT EXISTS (
    120          SELECT 1 FROM ref.lnk_substance2brand WHERE
    121                  fk_drug_product = (
    122                          SELECT pk FROM ref.drug_product WHERE
    123                                  description = s_id.brand_name
    124                                          AND
    125                                  preparation = s_id.form
    126                  )
    127                          AND
    128                  fk_substance = (
    129                          SELECT pk FROM ref.consumable_substance r_cs WHERE
    130                                  r_cs.description = s_id.substance
    131                                          AND
    132                                  r_cs.amount = s_id.strength
    133                                          AND
    134                                  r_cs.unit = s_id.unit
    135                  )
    136  );
    137
    138  -- ---------------------------------------------------------
    139  -- drop staging tables again, if needed --
    140  \\unset ON_ERROR_STOP
    141  DROP TABLE staging.india_drugs CASCADE;
    142  \set ON_ERROR_STOP 1
    143
    144  -- ---------------------------------------------------------
    145
    146  -- finalize transaction --
    147  -- uncomment this once you are satisfied your script works:
    148  COMMIT;
    149  -- ---------------------------------------------------------
    150  """
    151
    152  #---------------------------------------------------------------------------------------------------
    153 -def create_sql(filename):
    154
    155          csv_file = io.open(filename, mode = 'rt', encoding = 'utf8')
    156
    157          csv_lines = gmTools.unicode_csv_reader (
    158                  csv_file,
    159                  fieldnames = field_names,
    160                  delimiter = ';',
    161                  quotechar = '"',
    162                  dict = True
    163          )
    164
    165          print SQL_start
    166
    167          line_idx = 0
    168          skip_line = False
    169          for line in csv_lines:
    170                  line_idx += 1
    171                  print "-- line #%s" % line_idx
    172                  # normalize field content
    173                  for field in field_names:
    174                          try:
    175                                  line[field] = line[field].strip().strip(';,').strip().replace("'", "''")
    176                          except AttributeError:          # trailing fields are a list
    177                                  pass
    178                  # verify required fields
    179                  for field in non_empty_fields:
    180                          if line[field] == '':
    181                                  print "-- ignoring line: empty field [%s]" % field
    182                                  print "--", line
    183                                  print ""
    184                                  skip_line = True
    185                                  break
    186                  if skip_line:
    187                          skip_line = False
    188                          continue
    189                  # verify numeric fields
    190                  for field in numeric_fields:
    191                          if line[field] == '':
    192                                  continue
    193                          success, num_val = gmTools.input2decimal(initial = line[field])
    194                          if not success:
    195                                  print "-- ignoring line: field [%s] not numeric: >>>%s<<<" % (field, line[field])
    196                                  print "--", line
    197                                  print ""
    198                                  skip_line = True
    199                                  break
    200                          line[field] = num_val
    201                  if skip_line:
    202                          skip_line = False
    203                          continue
    204
    205                  # actually create SQL
    206                  # loop over strengths
    207                  for field in numeric_fields:
    208                          if line[field] == '':
    209                                  continue
    210                          line['brand_name'] = ('%%(product)s %%(%s)s (%%(company)s)' % field) % line
    211                          line['strength'] = line[field]
    212                          print SQL_stage_drug % line
    213
    214          print SQL_end
    215
    216  #---------------------------------------------------------------------------------------------------
    217
    218  create_sql(sys.argv[1])
    219