modelbase.py

Go to the documentation of this file.

#!/usr/bin/python
# -*- coding: utf-8 -*- 
 
import os
import datetime
import hashlib
import json
import base64
 
from pprint import pprint
 
import dateutil.parser
 
import pafera.db
 
from pafera.types import *
from pafera.utils import *
 
# *********************************************************************
class ModelBase(object):
  """Base class for all database models. Derived classes should add
  
   _dbfields   List of all model fields
   _dbindexes  List of indexes to add to the database
   _dblinks    List of linked objects for the database model manager
   _dbdisplay  List of default fields to display for the database
               model manager
   _dbflags    Enable security, tracking changes, and tracking changed
               values
  """
  
  SECURE_FIELDS  = [
    'dbowner', 
    'dbaccess', 
    'dbgroup', 
    'dbgroupaccess', 
    'dbaclid', 
  ]      
  
  # -------------------------------------------------------------------
  def __init__(self):
    """Initialize all fields at creation like a good programmer should
    """
    for k, v in self._dbfields.items():
      if v[0].find('PRIMARY KEY') != -1:
        setattr(self, k, None)
      elif v[0].find('INT') != -1:
        setattr(self, k, 0)
      elif v[0].find('FLOAT') != -1:
        setattr(self, k, 0.0)
      elif v[0] == 'DATETIME':
        setattr(self, k, datetime.datetime.now())
      else:
        setattr(self, k, '')
    
      self._changed = {}
  
  # -------------------------------------------------------------------
  def __hash__(self):
    """We take the easy route where the hash for an object is just its
    database ID.
    """
    return getattr(self, self._dbid)
    
  # -------------------------------------------------------------------
  def __eq__(self, o):
    """Two objects are equivalent if their models are the same and their
    IDs are the same.
    """
    return (self.__class__ ==  o.__class__) and (self.id == o.id) and (self.id and o.id)
    
  # -------------------------------------------------------------------  
  def __cmp__(self, o):
    """Comparing means simply subracting the database ids. Of course,
    this only works if you use integer IDs.
    """
    return getattr(self, self._dbid) - getattr(o, self._dbid)
    
  # -------------------------------------------------------------------
  def __str__(self):
    """Simply calls the toJSON() function.
    """
    return str(self.ToJSON())
    
  # -------------------------------------------------------------------
  def __repr__(self):
    """Simply outputs the toJSON() function with the model name.
    """
    return str(self.__class__) + '(' + str(self.ToJSON()) + ')'
  
  # -------------------------------------------------------------------
  def HasSameValues(self, o):
    """While operator = only checks class and ids, this checks every field
    to see if two objects or an object and a dict have the same values. 
    Handy for implementing copy-on-change systems  
    """
    if isinstance(o, dict):
      for k, v in self._dbfields.items():
        if getattr(self, k, None) != o.get(k, None):
          return 0
    else:
      for k, v in self._dbfields.items():
        if getattr(self, k, None) != getattr(o, k, None):
          return 0
    
    return 1    
  
  # -------------------------------------------------------------------
  def _HashPassword(self, password, salt):
    """Default password function. Override if you need more security.
    """
    return base64.b64encode(hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)).decode('ascii')
    
  # ------------------------------------------------------------------
  def Set(self, **kwargs):
    """We use this method instead of direct attribute access in order to 
    keep track of what values have been changed. Only updated values
    are saved to the database, increasing efficiency and output.
    
    The special field _updatechanged allows initialization of values
    without triggering changed fields. It's used by the library itself
    to turn database rows into initialized objects. Set it to 0 if you
    want to disable tracking, otherwise the default behavior will be 
    changed.
    """
    _updatechanged = kwargs.get('_updatechanged')
    
    if _updatechanged == None:
      _updatechanged  = 1
    
    for k, v in self._dbfields.items():
      if k not in kwargs:
        continue
      
      value   = kwargs[k]
      
      # Check if field has a validator
      if len(v) > 2:
        v[2](k, value)
      
      if _updatechanged:
        self._changed[k]  = getattr(self, k)
        
      if v[0].find('INT') != -1:
        setattr(self, k, int(value if value else 0))
      elif v[0].find('FLOAT') != -1:
        setattr(self, k, float(value if value else 0))
      elif v[0].find('NEWLINELIST') != -1:
        if value and not isinstance(value, list):
          value = [x.strip() for x in filter(None, value.split('\n'))]
        
        setattr(self, k, value if value else [])
      elif (v[0] == 'DICT' or v[0] == 'TRANSLATION') and isinstance(value, str):
        try:
          setattr(self, k, json.loads(value))
        except Exception as e:          
          setattr(self, k, {})
      elif v[0] == 'LIST' and isinstance(value, str):
        try:
          setattr(self, k, json.loads(value))
        except Exception as e:          
          setattr(self, k, [])
      elif v[0] == 'DATETIME':
        if isinstance(value, datetime.datetime):
          setattr(self, k, value)
        elif isinstance(value, int) or isinstance(value, float):
          setattr(self, k, datetime.datetime.fromtimestamp(value))
        else:
          setattr(self, k, dateutil.parser.parse(value))
      else:
        setattr(self, k, value)
        
    if self._dbflags & pafera.db.DB_SECURE:
      for r in self.SECURE_FIELDS:
        if r not in kwargs:
          continue
        
        v = fields[r]
        
        if getattr(self, r, None) is not v and updatechanged:
          self._changed[r]  = getattr(self, r)
          
        setattr(self, r, fields[r])
        
  # ------------------------------------------------------------------
  def UpdateFields(self, fieldnames):
    """This is a convenience method to update fields without going 
    through the Set() method. Useful for working on mutable
    collections like lists and dicts.
    
    fieldnames should be a list of fields separated by a comma and 
    space as in 'id, name, business'
    """
    fieldnames  = fieldnames.split(', ')
    
    for r in fieldnames:
      self._changed[r]  = getattr(self, r)
    
  # ------------------------------------------------------------------
  def OffsetTime(self, timeoffset):
    """Change all datetime fields to the new time offset. Useful for
    switching between local and GMT times.
    """
    for k, v in self._dbfields.items():
      if v[0] == 'DATETIME':
        d = getattr(self, k, None)
        
        if d:
          self.d  = datetime.datetime.fromtimestamp(d.timestamp() + timeoffset)
      
  # ------------------------------------------------------------------
  def SetPassword(self, field, password):
    """This special function hashes the password before saving the field.
    You should explicitly call this to avoid situations where you're 
    hashing an existing hash instead of the raw password itself.
    """
    salt  = os.urandom(32)
    key   = self._HashPassword(password, salt)
    
    setattr(self, field, base64.b64encode(salt).decode('ascii') + '' + key)
    self._changed[field]  = ''
    
  # ------------------------------------------------------------------
  def CheckPassword(self, field, password):
    """This special function checks to see if the password matches the 
    hash stored in the field.
    """
    oldpassword   = getattr(self, field)
    
    salt, hashed, empty  = oldpassword.split('=')
    
    salt    += '='
    salt    = base64.b64decode(salt.encode('ascii'))
    hashed  += '='
    
    return self._HashPassword(
      password, 
      salt
    ) == hashed
  
  # ------------------------------------------------------------------
  def ToJSON(self, fields = ''):
    """Converts this object into a format suitable for inclusion in JSON.
    """
    values  = {}
    
    if fields:
      fields  = fields.split(', ')
    
    for k, v in self._dbfields.items():
      if fields and k not in fields:
        continue
      
      if k == 'rid':
        rid = getattr(self, k)
        values['idcode']  = ToShortCode(rid) if rid else ''
      
      if 'PASSWORD' in v[0]:
        values[k] = ''
      elif 'DATETIME' in v[0]:
        values[k] = getattr(self, k).isoformat()
      else:
        values[k] = getattr(self, k)
      
    return values
  
  # ------------------------------------------------------------------
  def SetOwner(self, ownerid):
    """Special security functions that are useful only if you have enabled
    security in _dbflags for the model.
    """
    self.dbowner  = ownerid
    
  # ------------------------------------------------------------------
  def SetGroup(self, groupid):
    """Special security functions that are useful only if you have enabled
    security in _dbflags for the model.
    """
    self.dbgroup  = groupid
    
  # ------------------------------------------------------------------
  def SetAccess(self, access):
    """Special security functions that are useful only if you have enabled
    security in _dbflags for the model.
    """
    self.dbaccess  = access
    
  # ------------------------------------------------------------------
  def GetACL(self, db):
    """Since we store ACLs as a database lookup, be sure to use these getters
    and setters if you plan on using ACLs.
    """
    if not self.dbaclid:
      return {}
    
    if self.dbacl:
      return self.dbacl
    
    self.dbacl  = db.GetACL(self.dbaclid)
    
    return self.dbacl
  
  # ------------------------------------------------------------------
  def SetACL(self, db, acl):
    """ACLs in Pafera are defined as a set of rules similar to cascading
    style sheets. The default access for the model comes first, then 
    is overridden by group access, then overridden by user access.
    
    Note that for group access, if *one* group has access, then the
    values for the other groups are ignored.
    """
    if not acl or (not ArrayV(acl, 'users') and not ArrayV(acl, 'groups')):
      self.dbaclid  = 0
      self.dbacl    = {
        'users': {}, 
        'groups': {},
      }
      return
    
    self.dbacl    = acl
    
    acljson = json.dumps(acl, sort_keys = 1)
    
    r = db.Execute('SELECT id FROM system_acl WHERE rules = ?', acljson)
    
    if r:
      self.dbaclid  = r[0]
    else:
      self.dbaclid  = self.Execute('INSERT INTO system_acl(rules) VALUES(?)', acljson)
 
# *********************************************************************
class DBList(object):
  """Iterator class for database access that supports automatic chunking.
  Using this class, you can examine billions of rows of data without 
  worrying about memory usage."""
  
  # -------------------------------------------------------------------
  def __init__(self, db, model, **kwargs):
    """Same parameters as db.Find(), mainly because that function only
    sets up the parameters and lets this class do the heavy lifting.
    """
    super(DBList, self).__init__()
    
    params  = kwargs.get('params')
    
    if not isinstance(params, list):
      if params != None:
        params  = [params]
      else:
        params  = []
    
    self.db         = db
    self.model      = model
    self.cond       = StrV(kwargs, 'cond')
    self.params     = params
    self.orderby    = StrV(kwargs, 'orderby')
    self.fields     = StrV(kwargs, 'fields', '*')
    self.start      = IntV(kwargs, 'start', 0, 2147483647, 0)
    self.limit      = IntV(kwargs, 'limit', 1, 2147483647, 100)
    self.randompos  = 0
    self.pos        = -1
    self.length     = 0
    self.cachepos   = 0
    self.cache      = []
    self.randomlist = []
    
    self.enablesecurity = self.model._dbflags & pafera.db.DB_SECURE
    
  # -------------------------------------------------------------------
  def __iter__(self):
    """convenience functions to allow use in iterator loops.
    """
    self.pos  = -1
    return self
    
  # -------------------------------------------------------------------
  def __next__(self):
    """convenience functions to allow use in iterator loops.
    """
    if self.pos == -1:
      self.Count()
      self.pos  = self.start - 1
    
    if not self.length:
      raise StopIteration
    
    if self.pos < self.length - 1 and self.pos < (self.start + self.limit - 1):
      self.pos  +=  1
    else:
      raise StopIteration
      
    return self[self.pos]
    
  # -------------------------------------------------------------------
  def __bool__(self):
    """Conversion function to check if the dataset is empty.
    """
    if self.pos == -1:
      self.Count()
    
    return self.length != 0
    
  # -------------------------------------------------------------------
  def __repr__(self):
    """Returns the parameters for this iterator. Handy for debugging.
    """
    if self.pos == -1:
      self.Count()
      
    return f"""DBList({self.model})
  Filter: {self.cond}
  Params: {self.params}
  Size:   {self.length}
"""
  
  # -------------------------------------------------------------------
  def Count(self):
    """Returns the length of the underlying dataset.
    """
    if self.cond:
      if self.params:
        r = self.db.Execute(f"SELECT COUNT(*) FROM {self.model._dbtable} {self.cond}", self.params)[0]
      else:
        r = self.db.Execute(f"SELECT COUNT(*) FROM {self.model._dbtable} {self.cond}")[0]
    else:
      r = self.db.Execute(f"SELECT COUNT(*) FROM {self.model._dbtable}")[0]
      
    if r:
      self.length = int(r[0])
      
    return self.length
    
  # -------------------------------------------------------------------
  def Filter(self, cond, params):
    """Resets the condition and parameters for the query used. It saves a
    little bit of time as opposed to creating a new object.
    """
    self.cond     = cond
    self.params   = params
    self.cache    = []
    self.pos      = -1
    self.cachepos = 0
    return self
    
  # -------------------------------------------------------------------
  def OrderBy(self, order):
    """Resets the order for the query used. It saves a
    little bit of time as opposed to creating a new object.
    """
    self.orderby  = order
    self.cache    = []
    self.pos      = -1
    self.cachepos = 0
    return self
    
  # -------------------------------------------------------------------
  def __len__(self):
    """Total number of rows for the current query.
    """
    if not self.cache:
      self.Count()
    
    return self.length
    
  # -------------------------------------------------------------------
  def __getitem__(self, pos):
    """Returns the item at pos, retrieving it from the database if 
    necessary.
    """
    if self.randomlist:
      self.randompos  = self.randomlist[pos]
      self._RefreshCache(self.randompos)
      return self.cache[0]
    
    if (not self.cache 
      or pos < self.cachepos 
      or pos > self.cachepos + len(self.cache) - 1
    ):
      self._RefreshCache(pos)
    
    return self.cache[pos - self.cachepos]
    
  # -------------------------------------------------------------------
  def __delitem__(self, pos):
    """Deletes the item from the database. Note that this will alter
    the length of the dataset.
    """
    if self.randomlist:
      self.randompos  = self.randomlist[pos]
      self._RefreshCache(self.randompos)
      self.db.Delete(self.cache[self.randompos - self.cachepos])
      del self.cache[self.randompos - self.cachepos]
      self.length -=  1
      return self
    
    if not self.cache or pos < self.cachepos or pos > self.cachepos + len(self.cache) - 1:
      self._RefreshCache(pos)
    
    self.db.Delete(self.cache[pos - self.cachepos])
    del self.cache[pos - self.cachepos]
    self.length -=  1
    return self
    
  # -------------------------------------------------------------------
  def _RefreshCache(self, pos):
    """Fetch new data from the database. You should not need to call this
    directly as __getitem__() automatically does it for you.
    """
    query = f"SELECT {self.fields} FROM {self.model._dbtable} "
    
    if self.cond:
      query = query + self.cond
      
    if self.orderby:
      query = query + ' ORDER BY ' + self.orderby
    
      if ' LIMIT ' not in query:
        if self.randomlist:
          query = query + f" LIMIT {pos}, 1"
        else:
          query = query + f" LIMIT {pos}, {self.limit}"
    
    self.cache    = []
    
    for r in self.db.Query(query, self.params):
      if isinstance(self.model, pafera.modelbase.ModelBase):
        o = self.model.__class__()
      else:
        o = self.model()
        
      r['_updatechanged'] = 0
      
      o.Set(**r)
      
      if self.enablesecurity:
        access    = self.db.GetAccess(o)
        
        if not (access & pafera.db.DB_CAN_VIEW):
          continue
      
      self.cache.append(o)
      
    self.cachepos = pos
    
  # -------------------------------------------------------------------
  def SetRandom(self, active):
    """Enables random ordering for the returned elements. Note that this
    will cause severe database activity if you use it on a dataset that
    won't fit in memory, so is a convenience for small datasets in 
    local memory more than anything else.
    """
    if active:
      self.randomlist = list(range(0, self.Count()))
      random.shuffle(self.randomlist)
    else:
      self.randomlist = []