Listing integration

api/v1/endpoints/listings.py

@@ -43,23 +43,32 @@ class Listing(Resource):
     @api.marshal_with(new_listing)
     @api.response(201, constants.NEW_CANDIDATE_SUCCESS)
     @api.response(400, constants.MISSING_PAYLOAD_DATA)
+    @api.response(428, constants.INVALID_CANDIDATE_OR_POLL_CLOSED)
     @api.response(500, constants.SERVER_ERROR)
     def post(self):
         """
         Persist a new listing to file storage, db, and protocol
         """
+        is_datatrust = deployed.get_backend_address()
+        if not is_datatrust:
+            api.abort(500, 'This server is not the approved datatrust. New candidates not allowed')
         timings = {}
         start_time = time.time()
         payload = {}
         uploaded_md5 = None
+        data_hash = None
         for item in ['title', 'description', 'license', 'file_type', 'md5_sum', 'listing_hash']:
             if not request.form.get(item):
                 api.abort(400, (constants.MISSING_PAYLOAD_DATA % item))
             else:
                 payload[item] = request.form.get(item)
-        if request.form.get('tags'):
-            payload['tags'] = [x.strip() for x in request.form.get('tags').split(',')]
-        filenames = []
+                if request.form.get('tags'):
+                    payload['tags'] = [x.strip() for x in request.form.get('tags').split(',')]
+                    filenames = []
+        owner = deployed.validate_candidate(payload['listing_hash'])
+        if owner is None:
+            api.abort(428, constants.INVALID_CANDIDATE_OR_POLL_CLOSED)
+        payload['owner'] = owner
         md5_sum = request.form.get('md5_sum')
         if request.form.get('filenames'):
             filenames = request.form.get('filenames').split(',')
@@ -82,11 +91,17 @@ def post(self):
             with open(f'{destination}{filename}', 'rb') as data:
                 # apparently this overwrites existing files.
                 # something to think about?
-                s3.upload_fileobj(data, settings.S3_DESTINATION, filename)
+                s3_filename = f"{owner}/{constants.S3_CANDIDATE}/{filename}"
+                s3.upload_fileobj(data, settings.S3_DESTINATION, s3_filename)
             timings['s3_save'] = time.time() - local_finish
+            data_hash = deployed.create_file_hash(f'{destination}{filename}')
             os.remove(f'{destination}{filename}')
         log.info(timings)
         db = dynamo.dynamo_conn
         db_entry = db.add_listing(payload)
         if db_entry == constants.DB_SUCCESS:
-            return {'message': constants.NEW_CANDIDATE_SUCCESS}, 201
+            try:
+                deployed.send_data_hash(payload['listing_hash'], data_hash)
+                return {'message': constants.NEW_CANDIDATE_SUCCESS}, 201
+            except ValueError as err:
+                api.abort(428, err)

constants.py

@@ -2,6 +2,8 @@
 NEW_CANDIDATE_SUCCESS = 'Candidate successfully added'
 MISSING_PAYLOAD_DATA = 'Incomplete payload data in request: %s'
 SERVER_ERROR = 'Listing failed due to server side error: %s'
+INVALID_CANDIDATE_OR_POLL_CLOSED = 'Listing is not a candidate. Cannot set data hash until it is.'
+NOT_DATATRUST = 'Server is not the datatrust, unable to send data hash'
 
 # Database response messages
 DB_SUCCESS = 'Database transaction completed successfully'
@@ -11,3 +13,8 @@
 # Protocol constants
 PROTOCOL_APPLICATION = 1
 PROTOCOL_REGISTRATION = 4
+CANDIDATE_ADDED = 'CandidateAdded'
+
+# S3 Namespaces
+S3_CANDIDATE = 'candidate'
+S3_LISTING = 'listing'

protocol/__init__.py

@@ -3,7 +3,7 @@
 """
 import os
 import logging.config
-from time import sleep
+from time import sleep, time
 from web3 import Web3
 from computable.contracts import Datatrust, Voting
 from computable.helpers.transaction import call, send
@@ -57,9 +57,8 @@ def initialize_datatrust(self):
         self.voting = Voting(self.datatrust_wallet)
         self.voting.at(self.w3, self.voting_contract)
 
-        backend = call(self.datatrust.get_backend_address())
         datatrust_hash = self.w3.sha3(text=self.datatrust_host)
-        if backend == self.datatrust_wallet:
+        if self.get_backend_address():
             log.info('This server is the datatrust host. Resolving registration')
             resolve = send(
                 self.w3, 
@@ -103,6 +102,16 @@ def initialize_datatrust(self):
                 self.wait_for_mining(resolve)
                 register = self.register_host()
 
+    def get_backend_address(self):
+        """
+        Return True if the Ethereum address for the voted-in datatrust is this datatrust
+        """
+        backend = call(self.datatrust.get_backend_address())
+        if backend == self.datatrust_wallet:
+            return True
+        else:
+            return False
+
     def wait_for_vote(self):
         """
         Check if this backend is registered as the datatrust
@@ -137,15 +146,12 @@ def send_data_hash(self, listing, data_hash):
         """
         On a successful post to the API db, send the data hash to protocol
         """
-        datatrust_hash = self.w3.sha3(text=self.datatrust_host)
-        is_candidate = call(self.voting.is_candidate(datatrust_hash))
-        candidate_is = call(self.voting.candidate_is(datatrust_hash, constants.PROTOCOL_APPLICATION))
-        if is_candidate and candidate_is:    
+        if self.get_backend_address():   
             receipt = send(self.w3, self.datatrust_key, self.datatrust.set_data_hash(listing, data_hash))
             return receipt
         else:
-            log.critical('This server is not the datatrust, unable to send data hash')
-            raise ValueError('Server is not the datatrust, unable to send data hash')
+            log.critical(constants.NOT_DATATRUST)
+            raise ValueError(constants.NOT_DATATRUST)
 
     def wait_for_mining(self, tx):
         """
@@ -167,8 +173,35 @@ def create_file_hash(self, data):
         """
         sha3_hash = None
         with open(data, 'rb') as file_contents:
-            file_contents.read()
-            sha3_hash = self.w3.sha3(file_contents)
+            b = file_contents.read(1024*1024) # read file in 1MB chunks
+            while b:
+                sha3_hash = self.w3.sha3(b)
+                b = file_contents.read(1024*1024)
         return sha3_hash
 
+    def validate_candidate(self, listing_hash):
+        """
+        Verify a candidate has been submitted to Computable Protocol
+        by parsing logs for 'CandidateAdded' event and matching listing hash
+        Verify voteBy has not expired
+        :params listing_hash: String listing hash for listing
+        :return owner address if valid otherwise return None:
+        :return type string:
+        """
+        owner = None
+        voting_filter = self.voting.deployed.eventFilter(
+            constants.CANDIDATE_ADDED,{'fromBlock':0,'toBlock':'latest'}
+        )
+        events = voting_filter.get_all_entries()
+        for evt in events:
+            event_hash = '0x' + evt['args']['hash'].hex()
+            print(f'Comparing event hash {event_hash} to listing hash {listing_hash}')
+            if event_hash == listing_hash:
+                log.info(f'Listing {listing_hash} has been listed as a candidate in protocol')
+                voteBy = evt['args']['voteBy']
+                if voteBy > int(time.time()):
+                    owner = evt['args']['owner']
+        return owner
+
+
 deployed = Protocol()