sentinel/lib/models.py

import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'lib'))
import init
import time
import datetime
import re
import simplejson
from peewee import IntegerField, CharField, TextField, ForeignKeyField, DecimalField, DateTimeField
import peewee
import playhouse.signals
import misc
import dashd
from misc import (printdbg, is_numeric)
import config
from bitcoinrpc.authproxy import JSONRPCException
try:
    import urllib.parse as urlparse
except ImportError:
    import urlparse

# our mixin
from governance_class import GovernanceClass

db = config.db
db.connect()


# TODO: lookup table?
DASHD_GOVOBJ_TYPES = {
    'proposal': 1,
    'superblock': 2,
}
GOVOBJ_TYPE_STRINGS = {
    1: 'proposal',
    2: 'trigger',  # it should be trigger here, not superblock
}

# schema version follows format 'YYYYMMDD-NUM'.
#
# YYYYMMDD is the 4-digit year, 2-digit month and 2-digit day the schema
# changes were added.
#
# NUM is a numerical version of changes for that specific date. If the date
# changes, the NUM resets to 1.
SCHEMA_VERSION = '20170111-1'

# === models ===


class BaseModel(playhouse.signals.Model):
    class Meta:
        database = db

    @classmethod
    def is_database_connected(self):
        return not db.is_closed()


class GovernanceObject(BaseModel):
    parent_id = IntegerField(default=0)
    object_creation_time = IntegerField(default=int(time.time()))
    object_hash = CharField(max_length=64)
    object_parent_hash = CharField(default='0')
    object_type = IntegerField(default=0)
    object_revision = IntegerField(default=1)
    object_fee_tx = CharField(default='')
    yes_count = IntegerField(default=0)
    no_count = IntegerField(default=0)
    abstain_count = IntegerField(default=0)
    absolute_yes_count = IntegerField(default=0)

    class Meta:
        db_table = 'governance_objects'

    # sync dashd gobject list with our local relational DB backend
    @classmethod
    def sync(self, dashd):
        golist = dashd.rpc_command('gobject', 'list')

        # objects which are removed from the network should be removed from the DB
        try:
            for purged in self.purged_network_objects(list(golist.keys())):
                # SOMEDAY: possible archive step here
                purged.delete_instance(recursive=True, delete_nullable=True)
        except Exception as e:
            printdbg("Got an error while purging: %s" % e)

        for item in golist.values():
            try:
                (go, subobj) = self.import_gobject_from_dashd(dashd, item)
            except Exception as e:
                printdbg("Got an error upon import: %s" % e)

    @classmethod
    def purged_network_objects(self, network_object_hashes):
        query = self.select()
        if network_object_hashes:
            query = query.where(~(self.object_hash << network_object_hashes))
        return query

    @classmethod
    def import_gobject_from_dashd(self, dashd, rec):
        import decimal
        import dashlib
        import binascii
        import gobject_json

        object_hash = rec['Hash']

        gobj_dict = {
            'object_hash': object_hash,
            'object_fee_tx': rec['CollateralHash'],
            'absolute_yes_count': rec['AbsoluteYesCount'],
            'abstain_count': rec['AbstainCount'],
            'yes_count': rec['YesCount'],
            'no_count': rec['NoCount'],
        }

        # deserialise and extract object
        json_str = binascii.unhexlify(rec['DataHex']).decode('utf-8')
        dikt = gobject_json.extract_object(json_str)

        subobj = None

        type_class_map = {
            1: Proposal,
            2: Superblock,
        }
        subclass = type_class_map[dikt['type']]

        # set object_type in govobj table
        gobj_dict['object_type'] = subclass.govobj_type

        # exclude any invalid model data from dashd...
        valid_keys = subclass.serialisable_fields()
        subdikt = {k: dikt[k] for k in valid_keys if k in dikt}

        # get/create, then sync vote counts from dashd, with every run
        govobj, created = self.get_or_create(object_hash=object_hash, defaults=gobj_dict)
        if created:
            printdbg("govobj created = %s" % created)
        count = govobj.update(**gobj_dict).where(self.id == govobj.id).execute()
        if count:
            printdbg("govobj updated = %d" % count)
        subdikt['governance_object'] = govobj

        # get/create, then sync payment amounts, etc. from dashd - Dashd is the master
        try:
            newdikt = subdikt.copy()
            newdikt['object_hash'] = object_hash
            if subclass(**newdikt).is_valid() is False:
                govobj.vote_delete(dashd)
                return (govobj, None)

            subobj, created = subclass.get_or_create(object_hash=object_hash, defaults=subdikt)

        except Exception as e:
            # in this case, vote as delete, and log the vote in the DB
            printdbg("Got invalid object from dashd! %s" % e)
            govobj.vote_delete(dashd)
            return (govobj, None)

        if created:
            printdbg("subobj created = %s" % created)
        count = subobj.update(**subdikt).where(subclass.id == subobj.id).execute()
        if count:
            printdbg("subobj updated = %d" % count)

        # ATM, returns a tuple w/gov attributes and the govobj
        return (govobj, subobj)

    def vote_delete(self, dashd):
        if not self.voted_on(signal=VoteSignals.delete, outcome=VoteOutcomes.yes):
            self.vote(dashd, VoteSignals.delete, VoteOutcomes.yes)
        return

    def get_vote_command(self, signal, outcome):
        cmd = ['gobject', 'vote-conf', self.object_hash,
               signal.name, outcome.name]
        return cmd

    def vote(self, dashd, signal, outcome):
        import dashlib

        # At this point, will probably never reach here. But doesn't hurt to
        # have an extra check just in case objects get out of sync (people will
        # muck with the DB).
        if (self.object_hash == '0' or not misc.is_hash(self.object_hash)):
            printdbg("No governance object hash, nothing to vote on.")
            return

        # have I already voted on this gobject with this particular signal and outcome?
        if self.voted_on(signal=signal):
            printdbg("Found a vote for this gobject/signal...")
            vote = self.votes.where(Vote.signal == signal)[0]

            # if the outcome is the same, move on, nothing more to do
            if vote.outcome == outcome:
                # move on.
                printdbg("Already voted for this same gobject/signal/outcome, no need to re-vote.")
                return
            else:
                printdbg("Found a STALE vote for this gobject/signal, deleting so that we can re-vote.")
                vote.delete_instance()

        else:
            printdbg("Haven't voted on this gobject/signal yet...")

        # now ... vote!

        vote_command = self.get_vote_command(signal, outcome)
        printdbg(' '.join(vote_command))
        output = dashd.rpc_command(*vote_command)

        # extract vote output parsing to external lib
        voted = dashlib.did_we_vote(output)

        if voted:
            printdbg('VOTE success, saving Vote object to database')
            Vote(governance_object=self, signal=signal, outcome=outcome,
                 object_hash=self.object_hash).save()
        else:
            printdbg('VOTE failed, trying to sync with network vote')
            self.sync_network_vote(dashd, signal)

    def sync_network_vote(self, dashd, signal):
        printdbg('\tSyncing network vote for object %s with signal %s' % (self.object_hash, signal.name))
        vote_info = dashd.get_my_gobject_votes(self.object_hash)
        for vdikt in vote_info:
            if vdikt['signal'] != signal.name:
                continue

            # ensure valid outcome
            outcome = VoteOutcomes.get(vdikt['outcome'])
            if not outcome:
                continue

            printdbg('\tFound a matching valid vote on the network, outcome = %s' % vdikt['outcome'])
            Vote(governance_object=self, signal=signal, outcome=outcome,
                 object_hash=self.object_hash).save()

    def voted_on(self, **kwargs):
        signal = kwargs.get('signal', None)
        outcome = kwargs.get('outcome', None)

        query = self.votes

        if signal:
            query = query.where(Vote.signal == signal)

        if outcome:
            query = query.where(Vote.outcome == outcome)

        count = query.count()
        return count


class Setting(BaseModel):
    name = CharField(default='')
    value = CharField(default='')
    created_at = DateTimeField(default=datetime.datetime.utcnow())
    updated_at = DateTimeField(default=datetime.datetime.utcnow())

    class Meta:
        db_table = 'settings'


class Proposal(GovernanceClass, BaseModel):
    governance_object = ForeignKeyField(GovernanceObject, related_name='proposals', on_delete='CASCADE', on_update='CASCADE')
    name = CharField(default='', max_length=40)
    url = CharField(default='')
    start_epoch = IntegerField()
    end_epoch = IntegerField()
    payment_address = CharField(max_length=36)
    payment_amount = DecimalField(max_digits=16, decimal_places=8)
    object_hash = CharField(max_length=64)

    # src/governance-validators.cpp
    MAX_DATA_SIZE = 512

    govobj_type = DASHD_GOVOBJ_TYPES['proposal']

    class Meta:
        db_table = 'proposals'

    def is_valid(self):
        import dashlib

        printdbg("In Proposal#is_valid, for Proposal: %s" % self.__dict__)

        try:
            # proposal name exists and is not null/whitespace
            if (len(self.name.strip()) == 0):
                printdbg("\tInvalid Proposal name [%s], returning False" % self.name)
                return False

            # proposal name is normalized (something like "[a-zA-Z0-9-_]+")
            if not re.match(r'^[-_a-zA-Z0-9]+$', self.name):
                printdbg("\tInvalid Proposal name [%s] (does not match regex), returning False" % self.name)
                return False

            # end date < start date
            if (self.end_epoch <= self.start_epoch):
                printdbg("\tProposal end_epoch [%s] <= start_epoch [%s] , returning False" % (self.end_epoch, self.start_epoch))
                return False

            # amount must be numeric
            if misc.is_numeric(self.payment_amount) is False:
                printdbg("\tProposal amount [%s] is not valid, returning False" % self.payment_amount)
                return False

            # amount can't be negative or 0
            if (float(self.payment_amount) <= 0):
                printdbg("\tProposal amount [%s] is negative or zero, returning False" % self.payment_amount)
                return False

            # payment address is valid base58 dash addr, non-multisig
            if not dashlib.is_valid_address(self.payment_address, config.network):
                printdbg("\tPayment address [%s] not a valid Dash address for network [%s], returning False" % (self.payment_address, config.network))
                return False

            # URL
            if (len(self.url.strip()) < 4):
                printdbg("\tProposal URL [%s] too short, returning False" % self.url)
                return False

            # proposal URL has any whitespace
            if (re.search(r'\s', self.url)):
                printdbg("\tProposal URL [%s] has whitespace, returning False" % self.name)
                return False

            # Dash Core restricts proposals to 512 bytes max
            if len(self.serialise()) > (self.MAX_DATA_SIZE * 2):
                printdbg("\tProposal [%s] is too big, returning False" % self.name)
                return False

            try:
                parsed = urlparse.urlparse(self.url)
            except Exception as e:
                printdbg("\tUnable to parse Proposal URL, marking invalid: %s" % e)
                return False

        except Exception as e:
            printdbg("Unable to validate in Proposal#is_valid, marking invalid: %s" % e.message)
            return False

        printdbg("Leaving Proposal#is_valid, Valid = True")
        return True

    def is_expired(self, superblockcycle=None):
        from constants import SUPERBLOCK_FUDGE_WINDOW
        import dashlib

        if not superblockcycle:
            raise Exception("Required field superblockcycle missing.")

        printdbg("In Proposal#is_expired, for Proposal: %s" % self.__dict__)
        now = misc.now()
        printdbg("\tnow = %s" % now)

        # half the SB cycle, converted to seconds
        # add the fudge_window in seconds, defined elsewhere in Sentinel
        expiration_window_seconds = int(
            (dashlib.blocks_to_seconds(superblockcycle) / 2) +
            SUPERBLOCK_FUDGE_WINDOW
        )
        printdbg("\texpiration_window_seconds = %s" % expiration_window_seconds)

        # "fully expires" adds the expiration window to end time to ensure a
        # valid proposal isn't excluded from SB by cutting it too close
        fully_expires_at = self.end_epoch + expiration_window_seconds
        printdbg("\tfully_expires_at = %s" % fully_expires_at)

        if (fully_expires_at < now):
            printdbg("\tProposal end_epoch [%s] < now [%s] , returning True" % (self.end_epoch, now))
            return True

        printdbg("Leaving Proposal#is_expired, Expired = False")
        return False

    @classmethod
    def approved_and_ranked(self, proposal_quorum, next_superblock_max_budget):
        # return all approved proposals, in order of descending vote count
        #
        # we need a secondary 'order by' in case of a tie on vote count, since
        # superblocks must be deterministic
        query = (self
                 .select(self, GovernanceObject)  # Note that we are selecting both models.
                 .join(GovernanceObject)
                 .where(GovernanceObject.absolute_yes_count > proposal_quorum)
                 .order_by(GovernanceObject.absolute_yes_count.desc(), GovernanceObject.object_hash.desc())
                 )

        ranked = []
        for proposal in query:
            proposal.max_budget = next_superblock_max_budget
            if proposal.is_valid():
                ranked.append(proposal)

        return ranked

    @classmethod
    def expired(self, superblockcycle=None):
        if not superblockcycle:
            raise Exception("Required field superblockcycle missing.")

        expired = []

        for proposal in self.select():
            if proposal.is_expired(superblockcycle):
                expired.append(proposal)

        return expired

    @property
    def rank(self):
        rank = 0
        if self.governance_object:
            rank = self.governance_object.absolute_yes_count
            return rank


class Superblock(BaseModel, GovernanceClass):
    governance_object = ForeignKeyField(GovernanceObject, related_name='superblocks', on_delete='CASCADE', on_update='CASCADE')
    event_block_height = IntegerField()
    payment_addresses = TextField()
    payment_amounts = TextField()
    proposal_hashes = TextField(default='')
    sb_hash = CharField()
    object_hash = CharField(max_length=64)

    govobj_type = DASHD_GOVOBJ_TYPES['superblock']
    only_masternode_can_submit = True

    class Meta:
        db_table = 'superblocks'

    def is_valid(self):
        import dashlib
        import decimal

        printdbg("In Superblock#is_valid, for SB: %s" % self.__dict__)

        # it's a string from the DB...
        addresses = self.payment_addresses.split('|')
        for addr in addresses:
            if not dashlib.is_valid_address(addr, config.network):
                printdbg("\tInvalid address [%s], returning False" % addr)
                return False

        amounts = self.payment_amounts.split('|')
        for amt in amounts:
            if not misc.is_numeric(amt):
                printdbg("\tAmount [%s] is not numeric, returning False" % amt)
                return False

            # no negative or zero amounts allowed
            damt = decimal.Decimal(amt)
            if not damt > 0:
                printdbg("\tAmount [%s] is zero or negative, returning False" % damt)
                return False

        # verify proposal hashes correctly formatted...
        if len(self.proposal_hashes) > 0:
            hashes = self.proposal_hashes.split('|')
            for object_hash in hashes:
                if not misc.is_hash(object_hash):
                    printdbg("\tInvalid proposal hash [%s], returning False" % object_hash)
                    return False

        # ensure number of payment addresses matches number of payments
        if len(addresses) != len(amounts):
            printdbg("\tNumber of payment addresses [%s] != number of payment amounts [%s], returning False" % (len(addresses), len(amounts)))
            return False

        printdbg("Leaving Superblock#is_valid, Valid = True")
        return True

    def hash(self):
        import dashlib
        return dashlib.hashit(self.serialise())

    def hex_hash(self):
        return "%x" % self.hash()

    # workaround for now, b/c we must uniquely ID a superblock with the hash,
    # in case of differing superblocks
    #
    # this prevents sb_hash from being added to the serialised fields
    @classmethod
    def serialisable_fields(self):
        return [
            'event_block_height',
            'payment_addresses',
            'payment_amounts',
            'proposal_hashes'
        ]

    # has this masternode voted to fund *any* superblocks at the given
    # event_block_height?
    @classmethod
    def is_voted_funding(self, ebh):
        count = (self.select()
                 .where(self.event_block_height == ebh)
                 .join(GovernanceObject)
                 .join(Vote)
                 .join(Signal)
                 .switch(Vote)  # switch join query context back to Vote
                 .join(Outcome)
                 .where(Vote.signal == VoteSignals.funding)
                 .where(Vote.outcome == VoteOutcomes.yes)
                 .count())
        return count

    @classmethod
    def latest(self):
        try:
            obj = self.select().order_by(self.event_block_height).desc().limit(1)[0]
        except IndexError as e:
            obj = None
        return obj

    @classmethod
    def at_height(self, ebh):
        query = (self.select().where(self.event_block_height == ebh))
        return query

    @classmethod
    def find_highest_deterministic(self, sb_hash):
        # highest block hash wins
        query = (self.select()
                 .where(self.sb_hash == sb_hash)
                 .order_by(self.object_hash.desc()))
        try:
            obj = query.limit(1)[0]
        except IndexError as e:
            obj = None
        return obj


# ok, this is an awkward way to implement these...
# "hook" into the Superblock model and run this code just before any save()
from playhouse.signals import pre_save


@pre_save(sender=Superblock)
def on_save_handler(model_class, instance, created):
    instance.sb_hash = instance.hex_hash()


class Signal(BaseModel):
    name = CharField(unique=True)
    created_at = DateTimeField(default=datetime.datetime.utcnow())
    updated_at = DateTimeField(default=datetime.datetime.utcnow())

    class Meta:
        db_table = 'signals'


class Outcome(BaseModel):
    name = CharField(unique=True)
    created_at = DateTimeField(default=datetime.datetime.utcnow())
    updated_at = DateTimeField(default=datetime.datetime.utcnow())

    class Meta:
        db_table = 'outcomes'


class Vote(BaseModel):
    governance_object = ForeignKeyField(GovernanceObject, related_name='votes', on_delete='CASCADE', on_update='CASCADE')
    signal = ForeignKeyField(Signal, related_name='votes', on_delete='CASCADE', on_update='CASCADE')
    outcome = ForeignKeyField(Outcome, related_name='votes', on_delete='CASCADE', on_update='CASCADE')
    voted_at = DateTimeField(default=datetime.datetime.utcnow())
    created_at = DateTimeField(default=datetime.datetime.utcnow())
    updated_at = DateTimeField(default=datetime.datetime.utcnow())
    object_hash = CharField(max_length=64)

    class Meta:
        db_table = 'votes'


class Transient(object):

    def __init__(self, **kwargs):
        for key in ['created_at', 'timeout', 'value']:
            self.__setattr__(key, kwargs.get(key))

    def is_expired(self):
        return (self.created_at + self.timeout) < misc.now()

    @classmethod
    def deserialise(self, json):
        try:
            dikt = simplejson.loads(json)
        # a no-op, but this tells us what exception to expect
        except simplejson.scanner.JSONDecodeError as e:
            raise e

        lizt = [dikt.get(key, None) for key in ['timeout', 'value']]
        lizt = list(set(lizt))
        if None in lizt:
            printdbg("Not all fields required for transient -- moving along.")
            raise Exception("Required fields not present for transient.")

        return dikt

    @classmethod
    def from_setting(self, setting):
        dikt = Transient.deserialise(setting.value)
        dikt['created_at'] = int((setting.created_at - datetime.datetime.utcfromtimestamp(0)).total_seconds())
        return Transient(**dikt)

    @classmethod
    def cleanup(self):
        for s in Setting.select().where(Setting.name.startswith('__transient_')):
            try:
                t = Transient.from_setting(s)
            except:
                continue

            if t.is_expired():
                s.delete_instance()

    @classmethod
    def get(self, name):
        setting_name = "__transient_%s" % (name)

        try:
            the_setting = Setting.get(Setting.name == setting_name)
            t = Transient.from_setting(the_setting)
        except Setting.DoesNotExist as e:
            return False

        if t.is_expired():
            the_setting.delete_instance()
            return False
        else:
            return t.value

    @classmethod
    def set(self, name, value, timeout):
        setting_name = "__transient_%s" % (name)
        setting_dikt = {
            'value': simplejson.dumps({
                'value': value,
                'timeout': timeout,
            }),
        }
        setting, created = Setting.get_or_create(name=setting_name, defaults=setting_dikt)
        return setting

    @classmethod
    def delete(self, name):
        setting_name = "__transient_%s" % (name)
        try:
            s = Setting.get(Setting.name == setting_name)
        except Setting.DoesNotExist as e:
            return False
        return s.delete_instance()

# === /models ===


def load_db_seeds():
    rows_created = 0

    for name in ['funding', 'valid', 'delete']:
        (obj, created) = Signal.get_or_create(name=name)
        if created:
            rows_created = rows_created + 1

    for name in ['yes', 'no', 'abstain']:
        (obj, created) = Outcome.get_or_create(name=name)
        if created:
            rows_created = rows_created + 1

    return rows_created


def db_models():
    """ Return a list of Sentinel DB models. """
    models = [
        GovernanceObject,
        Setting,
        Proposal,
        Superblock,
        Signal,
        Outcome,
        Vote
    ]
    return models


def check_db_sane():
    """ Ensure DB tables exist, create them if they don't. """
    check_db_schema_version()

    missing_table_models = []

    for model in db_models():
        if not getattr(model, 'table_exists')():
            missing_table_models.append(model)
            printdbg("[warning]: Table for %s (%s) doesn't exist in DB." % (model, model._meta.db_table))

    if missing_table_models:
        printdbg("[warning]: Missing database tables. Auto-creating tables.")
        try:
            db.create_tables(missing_table_models, safe=True)
        except (peewee.InternalError, peewee.OperationalError, peewee.ProgrammingError) as e:
            print("[error] Could not create tables: %s" % e)

    update_schema_version()
    purge_invalid_amounts()


def check_db_schema_version():
    """ Ensure DB schema is correct version. Drop tables if not. """
    db_schema_version = None

    try:
        db_schema_version = Setting.get(Setting.name == 'DB_SCHEMA_VERSION').value
    except (peewee.OperationalError, peewee.DoesNotExist, peewee.ProgrammingError) as e:
        printdbg("[info]: Can't get DB_SCHEMA_VERSION...")

    printdbg("[info]: SCHEMA_VERSION (code) = [%s]" % SCHEMA_VERSION)
    printdbg("[info]: DB_SCHEMA_VERSION = [%s]" % db_schema_version)
    if (SCHEMA_VERSION != db_schema_version):
        printdbg("[info]: Schema version mis-match. Syncing tables.")
        try:
            existing_table_names = db.get_tables()
            existing_models = [m for m in db_models() if m._meta.db_table in existing_table_names]
            if (existing_models):
                printdbg("[info]: Dropping tables...")
                db.drop_tables(existing_models, safe=False, cascade=False)
        except (peewee.InternalError, peewee.OperationalError, peewee.ProgrammingError) as e:
            print("[error] Could not drop tables: %s" % e)


def update_schema_version():
    schema_version_setting, created = Setting.get_or_create(name='DB_SCHEMA_VERSION', defaults={'value': SCHEMA_VERSION})
    if (schema_version_setting.value != SCHEMA_VERSION):
        schema_version_setting.save()
    return


def purge_invalid_amounts():
    result_set = Proposal.select(
        Proposal.id,
        Proposal.governance_object
    ).where(Proposal.payment_amount.contains(','))

    for proposal in result_set:
        gobject = GovernanceObject.get(
            GovernanceObject.id == proposal.governance_object_id
        )
        printdbg("[info]: Pruning governance object w/invalid amount: %s" % gobject.object_hash)
        gobject.delete_instance(recursive=True, delete_nullable=True)


# sanity checks...
check_db_sane()     # ensure tables exist
load_db_seeds()     # ensure seed data loaded

# convenience accessors
VoteSignals = misc.Bunch(**{sig.name: sig for sig in Signal.select()})
VoteOutcomes = misc.Bunch(**{out.name: out for out in Outcome.select()})