diff --git a/api/v1/endpoints/listings.py b/api/v1/endpoints/listings.py index 0e1237d..53eb91b 100644 --- a/api/v1/endpoints/listings.py +++ b/api/v1/endpoints/listings.py @@ -43,23 +43,32 @@ class Listing(Resource): @api.marshal_with(new_listing) @api.response(201, constants.NEW_CANDIDATE_SUCCESS) @api.response(400, constants.MISSING_PAYLOAD_DATA) + @api.response(428, constants.INVALID_CANDIDATE_OR_POLL_CLOSED) @api.response(500, constants.SERVER_ERROR) def post(self): """ Persist a new listing to file storage, db, and protocol """ + is_datatrust = deployed.get_backend_address() + if not is_datatrust: + api.abort(500, 'This server is not the approved datatrust. New candidates not allowed') timings = {} start_time = time.time() payload = {} uploaded_md5 = None + data_hash = None for item in ['title', 'description', 'license', 'file_type', 'md5_sum', 'listing_hash']: if not request.form.get(item): api.abort(400, (constants.MISSING_PAYLOAD_DATA % item)) else: payload[item] = request.form.get(item) - if request.form.get('tags'): - payload['tags'] = [x.strip() for x in request.form.get('tags').split(',')] - filenames = [] + if request.form.get('tags'): + payload['tags'] = [x.strip() for x in request.form.get('tags').split(',')] + filenames = [] + owner = deployed.validate_candidate(payload['listing_hash']) + if owner is None: + api.abort(428, constants.INVALID_CANDIDATE_OR_POLL_CLOSED) + payload['owner'] = owner md5_sum = request.form.get('md5_sum') if request.form.get('filenames'): filenames = request.form.get('filenames').split(',') @@ -82,11 +91,17 @@ def post(self): with open(f'{destination}{filename}', 'rb') as data: # apparently this overwrites existing files. # something to think about? - s3.upload_fileobj(data, settings.S3_DESTINATION, filename) + s3_filename = f"{owner}/{constants.S3_CANDIDATE}/{filename}" + s3.upload_fileobj(data, settings.S3_DESTINATION, s3_filename) timings['s3_save'] = time.time() - local_finish + data_hash = deployed.create_file_hash(f'{destination}{filename}') os.remove(f'{destination}{filename}') log.info(timings) db = dynamo.dynamo_conn db_entry = db.add_listing(payload) if db_entry == constants.DB_SUCCESS: - return {'message': constants.NEW_CANDIDATE_SUCCESS}, 201 + try: + deployed.send_data_hash(payload['listing_hash'], data_hash) + return {'message': constants.NEW_CANDIDATE_SUCCESS}, 201 + except ValueError as err: + api.abort(428, err) diff --git a/constants.py b/constants.py index 806d7b3..0d0e061 100644 --- a/constants.py +++ b/constants.py @@ -2,6 +2,8 @@ NEW_CANDIDATE_SUCCESS = 'Candidate successfully added' MISSING_PAYLOAD_DATA = 'Incomplete payload data in request: %s' SERVER_ERROR = 'Listing failed due to server side error: %s' +INVALID_CANDIDATE_OR_POLL_CLOSED = 'Listing is not a candidate. Cannot set data hash until it is.' +NOT_DATATRUST = 'Server is not the datatrust, unable to send data hash' # Database response messages DB_SUCCESS = 'Database transaction completed successfully' @@ -11,3 +13,8 @@ # Protocol constants PROTOCOL_APPLICATION = 1 PROTOCOL_REGISTRATION = 4 +CANDIDATE_ADDED = 'CandidateAdded' + +# S3 Namespaces +S3_CANDIDATE = 'candidate' +S3_LISTING = 'listing' \ No newline at end of file diff --git a/protocol/__init__.py b/protocol/__init__.py index 1710b26..8ca9c13 100644 --- a/protocol/__init__.py +++ b/protocol/__init__.py @@ -3,7 +3,7 @@ """ import os import logging.config -from time import sleep +from time import sleep, time from web3 import Web3 from computable.contracts import Datatrust, Voting from computable.helpers.transaction import call, send @@ -57,9 +57,8 @@ def initialize_datatrust(self): self.voting = Voting(self.datatrust_wallet) self.voting.at(self.w3, self.voting_contract) - backend = call(self.datatrust.get_backend_address()) datatrust_hash = self.w3.sha3(text=self.datatrust_host) - if backend == self.datatrust_wallet: + if self.get_backend_address(): log.info('This server is the datatrust host. Resolving registration') resolve = send( self.w3, @@ -103,6 +102,16 @@ def initialize_datatrust(self): self.wait_for_mining(resolve) register = self.register_host() + def get_backend_address(self): + """ + Return True if the Ethereum address for the voted-in datatrust is this datatrust + """ + backend = call(self.datatrust.get_backend_address()) + if backend == self.datatrust_wallet: + return True + else: + return False + def wait_for_vote(self): """ Check if this backend is registered as the datatrust @@ -137,15 +146,12 @@ def send_data_hash(self, listing, data_hash): """ On a successful post to the API db, send the data hash to protocol """ - datatrust_hash = self.w3.sha3(text=self.datatrust_host) - is_candidate = call(self.voting.is_candidate(datatrust_hash)) - candidate_is = call(self.voting.candidate_is(datatrust_hash, constants.PROTOCOL_APPLICATION)) - if is_candidate and candidate_is: + if self.get_backend_address(): receipt = send(self.w3, self.datatrust_key, self.datatrust.set_data_hash(listing, data_hash)) return receipt else: - log.critical('This server is not the datatrust, unable to send data hash') - raise ValueError('Server is not the datatrust, unable to send data hash') + log.critical(constants.NOT_DATATRUST) + raise ValueError(constants.NOT_DATATRUST) def wait_for_mining(self, tx): """ @@ -167,8 +173,35 @@ def create_file_hash(self, data): """ sha3_hash = None with open(data, 'rb') as file_contents: - file_contents.read() - sha3_hash = self.w3.sha3(file_contents) + b = file_contents.read(1024*1024) # read file in 1MB chunks + while b: + sha3_hash = self.w3.sha3(b) + b = file_contents.read(1024*1024) return sha3_hash + def validate_candidate(self, listing_hash): + """ + Verify a candidate has been submitted to Computable Protocol + by parsing logs for 'CandidateAdded' event and matching listing hash + Verify voteBy has not expired + :params listing_hash: String listing hash for listing + :return owner address if valid otherwise return None: + :return type string: + """ + owner = None + voting_filter = self.voting.deployed.eventFilter( + constants.CANDIDATE_ADDED,{'fromBlock':0,'toBlock':'latest'} + ) + events = voting_filter.get_all_entries() + for evt in events: + event_hash = '0x' + evt['args']['hash'].hex() + print(f'Comparing event hash {event_hash} to listing hash {listing_hash}') + if event_hash == listing_hash: + log.info(f'Listing {listing_hash} has been listed as a candidate in protocol') + voteBy = evt['args']['voteBy'] + if voteBy > int(time.time()): + owner = evt['args']['owner'] + return owner + + deployed = Protocol()