From e4579eccba71a75bdaf1af906c4863d51fa98986 Mon Sep 17 00:00:00 2001 From: Will Button Date: Mon, 5 Aug 2019 16:20:29 -0700 Subject: [PATCH 1/6] extract "Are you my datatrust" to function --- protocol/__init__.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/protocol/__init__.py b/protocol/__init__.py index 1710b26..9c76467 100644 --- a/protocol/__init__.py +++ b/protocol/__init__.py @@ -57,9 +57,7 @@ def initialize_datatrust(self): self.voting = Voting(self.datatrust_wallet) self.voting.at(self.w3, self.voting_contract) - backend = call(self.datatrust.get_backend_address()) - datatrust_hash = self.w3.sha3(text=self.datatrust_host) - if backend == self.datatrust_wallet: + if get_backend_address(): log.info('This server is the datatrust host. Resolving registration') resolve = send( self.w3, @@ -103,6 +101,16 @@ def initialize_datatrust(self): self.wait_for_mining(resolve) register = self.register_host() + def get_backend_address(self): + """ + Return True if the Ethereum address for the voted-in datatrust is this datatrust + """ + backend = call(self.datatrust.get_backend_address()) + if backend == self.datatrust_wallet: + return True + else: + return False + def wait_for_vote(self): """ Check if this backend is registered as the datatrust From 725612f5f8d0028e61b870fcc5c6fe4767c39d8c Mon Sep 17 00:00:00 2001 From: Will Button Date: Sun, 11 Aug 2019 14:58:30 -0700 Subject: [PATCH 2/6] new constants --- constants.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/constants.py b/constants.py index 806d7b3..77ee1ca 100644 --- a/constants.py +++ b/constants.py @@ -2,6 +2,8 @@ NEW_CANDIDATE_SUCCESS = 'Candidate successfully added' MISSING_PAYLOAD_DATA = 'Incomplete payload data in request: %s' SERVER_ERROR = 'Listing failed due to server side error: %s' +INVALID_CANDIDATE_OR_POLL_CLOSED = 'Listing is not a candidate. Cannot set data hash until it is.' +NOT_DATATRUST = 'Server is not the datatrust, unable to send data hash' # Database response messages DB_SUCCESS = 'Database transaction completed successfully' @@ -11,3 +13,7 @@ # Protocol constants PROTOCOL_APPLICATION = 1 PROTOCOL_REGISTRATION = 4 + +# S3 Namespaces +S3_CANDIDATE = 'candidate' +S3_LISTING = 'listing' \ No newline at end of file From 91f0f2ea941a51157171172f4eb695bf547d9436 Mon Sep 17 00:00:00 2001 From: Will Button Date: Sun, 11 Aug 2019 14:59:43 -0700 Subject: [PATCH 3/6] bucketize S3, check datatrust, send data hash --- api/v1/endpoints/listings.py | 24 +++++++++++++++++++++--- protocol/__init__.py | 28 +++++++++++++++++----------- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/api/v1/endpoints/listings.py b/api/v1/endpoints/listings.py index 0e1237d..93745c3 100644 --- a/api/v1/endpoints/listings.py +++ b/api/v1/endpoints/listings.py @@ -43,16 +43,28 @@ class Listing(Resource): @api.marshal_with(new_listing) @api.response(201, constants.NEW_CANDIDATE_SUCCESS) @api.response(400, constants.MISSING_PAYLOAD_DATA) + @api.response(428, constants.INVALID_CANDIDATE_OR_POLL_CLOSED) @api.response(500, constants.SERVER_ERROR) def post(self): """ Persist a new listing to file storage, db, and protocol """ + # Random thoughts about setting the data hash + # (done) 1. This datatrust must be approved and voted in. Do this first. + is_datatrust = deployed.get_backend_address() + if not is_datatrust: + api.abort(500, 'This server is not the approved datatrust. New candidates not allowed') + # 2. Only set the data hash after we know it is stored in S3 and dynamo + # (done) 3. Maybe s3 should be sub-divided: s3://owner/candidate and s3://owner/listings + # 4. Once a candidate is voted in, it gets moved to listing + # 5. If a candidate doesn't get voted in within the voting window, it is removed + # 6. That will require a celery worker and tracking candidates timings = {} start_time = time.time() payload = {} uploaded_md5 = None - for item in ['title', 'description', 'license', 'file_type', 'md5_sum', 'listing_hash']: + data_hash = None + for item in ['owner', 'title', 'description', 'license', 'file_type', 'md5_sum', 'listing_hash']: if not request.form.get(item): api.abort(400, (constants.MISSING_PAYLOAD_DATA % item)) else: @@ -82,11 +94,17 @@ def post(self): with open(f'{destination}{filename}', 'rb') as data: # apparently this overwrites existing files. # something to think about? - s3.upload_fileobj(data, settings.S3_DESTINATION, filename) + s3_filename = f"{payload['owner']}/{constants.S3_CANDIDATE}/{filename}" + s3.upload_fileobj(data, settings.S3_DESTINATION, s3_filename) timings['s3_save'] = time.time() - local_finish + data_hash = deployed.create_file_hash(f'{destination}{filename}') os.remove(f'{destination}{filename}') log.info(timings) db = dynamo.dynamo_conn db_entry = db.add_listing(payload) if db_entry == constants.DB_SUCCESS: - return {'message': constants.NEW_CANDIDATE_SUCCESS}, 201 + try: + deployed.send_data_hash(payload['listing_hash'], data_hash) + return {'message': constants.NEW_CANDIDATE_SUCCESS}, 201 + except ValueError as err: + api.abort(428, err) diff --git a/protocol/__init__.py b/protocol/__init__.py index 9c76467..7658842 100644 --- a/protocol/__init__.py +++ b/protocol/__init__.py @@ -57,7 +57,8 @@ def initialize_datatrust(self): self.voting = Voting(self.datatrust_wallet) self.voting.at(self.w3, self.voting_contract) - if get_backend_address(): + datatrust_hash = self.w3.sha3(text=self.datatrust_host) + if self.get_backend_address(): log.info('This server is the datatrust host. Resolving registration') resolve = send( self.w3, @@ -145,15 +146,18 @@ def send_data_hash(self, listing, data_hash): """ On a successful post to the API db, send the data hash to protocol """ - datatrust_hash = self.w3.sha3(text=self.datatrust_host) - is_candidate = call(self.voting.is_candidate(datatrust_hash)) - candidate_is = call(self.voting.candidate_is(datatrust_hash, constants.PROTOCOL_APPLICATION)) - if is_candidate and candidate_is: - receipt = send(self.w3, self.datatrust_key, self.datatrust.set_data_hash(listing, data_hash)) - return receipt + is_candidate = call(self.voting.is_candidate(listing)) + candidate_is = call(self.voting.candidate_is(listing, constants.PROTOCOL_APPLICATION)) + if self.get_backend_address(): + if is_candidate and candidate_is: + receipt = send(self.w3, self.datatrust_key, self.datatrust.set_data_hash(listing, data_hash)) + return receipt + else: + log.critical(constants.INVALID_CANDIDATE_OR_POLL_CLOSED) + raise ValueError(constants.INVALID_CANDIDATE_OR_POLL_CLOSED) else: - log.critical('This server is not the datatrust, unable to send data hash') - raise ValueError('Server is not the datatrust, unable to send data hash') + log.critical(constants.NOT_DATATRUST) + raise ValueError(constants.NOT_DATATRUST) def wait_for_mining(self, tx): """ @@ -175,8 +179,10 @@ def create_file_hash(self, data): """ sha3_hash = None with open(data, 'rb') as file_contents: - file_contents.read() - sha3_hash = self.w3.sha3(file_contents) + b = file_contents.read(1024*1024) # read file in 1MB chunks + while b: + sha3_hash = self.w3.sha3(b) + b = file_contents.read(1024*1024) return sha3_hash deployed = Protocol() From 610eb1a53eb14012807c28257485aa38c5886145 Mon Sep 17 00:00:00 2001 From: Will Button Date: Sun, 11 Aug 2019 16:17:50 -0700 Subject: [PATCH 4/6] Validate a listing candidate has been added in protocol --- api/v1/endpoints/listings.py | 21 +++++++++------------ constants.py | 1 + protocol/__init__.py | 27 ++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/api/v1/endpoints/listings.py b/api/v1/endpoints/listings.py index 93745c3..53eb91b 100644 --- a/api/v1/endpoints/listings.py +++ b/api/v1/endpoints/listings.py @@ -49,29 +49,26 @@ def post(self): """ Persist a new listing to file storage, db, and protocol """ - # Random thoughts about setting the data hash - # (done) 1. This datatrust must be approved and voted in. Do this first. is_datatrust = deployed.get_backend_address() if not is_datatrust: api.abort(500, 'This server is not the approved datatrust. New candidates not allowed') - # 2. Only set the data hash after we know it is stored in S3 and dynamo - # (done) 3. Maybe s3 should be sub-divided: s3://owner/candidate and s3://owner/listings - # 4. Once a candidate is voted in, it gets moved to listing - # 5. If a candidate doesn't get voted in within the voting window, it is removed - # 6. That will require a celery worker and tracking candidates timings = {} start_time = time.time() payload = {} uploaded_md5 = None data_hash = None - for item in ['owner', 'title', 'description', 'license', 'file_type', 'md5_sum', 'listing_hash']: + for item in ['title', 'description', 'license', 'file_type', 'md5_sum', 'listing_hash']: if not request.form.get(item): api.abort(400, (constants.MISSING_PAYLOAD_DATA % item)) else: payload[item] = request.form.get(item) - if request.form.get('tags'): - payload['tags'] = [x.strip() for x in request.form.get('tags').split(',')] - filenames = [] + if request.form.get('tags'): + payload['tags'] = [x.strip() for x in request.form.get('tags').split(',')] + filenames = [] + owner = deployed.validate_candidate(payload['listing_hash']) + if owner is None: + api.abort(428, constants.INVALID_CANDIDATE_OR_POLL_CLOSED) + payload['owner'] = owner md5_sum = request.form.get('md5_sum') if request.form.get('filenames'): filenames = request.form.get('filenames').split(',') @@ -94,7 +91,7 @@ def post(self): with open(f'{destination}{filename}', 'rb') as data: # apparently this overwrites existing files. # something to think about? - s3_filename = f"{payload['owner']}/{constants.S3_CANDIDATE}/{filename}" + s3_filename = f"{owner}/{constants.S3_CANDIDATE}/{filename}" s3.upload_fileobj(data, settings.S3_DESTINATION, s3_filename) timings['s3_save'] = time.time() - local_finish data_hash = deployed.create_file_hash(f'{destination}{filename}') diff --git a/constants.py b/constants.py index 77ee1ca..0d0e061 100644 --- a/constants.py +++ b/constants.py @@ -13,6 +13,7 @@ # Protocol constants PROTOCOL_APPLICATION = 1 PROTOCOL_REGISTRATION = 4 +CANDIDATE_ADDED = 'CandidateAdded' # S3 Namespaces S3_CANDIDATE = 'candidate' diff --git a/protocol/__init__.py b/protocol/__init__.py index 7658842..9707127 100644 --- a/protocol/__init__.py +++ b/protocol/__init__.py @@ -3,7 +3,7 @@ """ import os import logging.config -from time import sleep +from time import sleep, time from web3 import Web3 from computable.contracts import Datatrust, Voting from computable.helpers.transaction import call, send @@ -185,4 +185,29 @@ def create_file_hash(self, data): b = file_contents.read(1024*1024) return sha3_hash + def validate_candidate(self, listing_hash): + """ + Verify a candidate has been submitted to Computable Protocol + by parsing logs for 'CandidateAdded' event and matching listing hash + Verify voteBy has not expired + :params listing_hash: String listing hash for listing + :return owner address if valid otherwise return None: + :return type string: + """ + owner = None + voting_filter = self.voting.deployed.eventFilter( + constants.CANDIDATE_ADDED,{'fromBlock':0,'toBlock':'latest'} + ) + events = voting_filter.get_all_entries() + for evt in events: + event_hash = '0x' + evt['args']['hash'].hex() + print(f'Comparing event hash {event_hash} to listing hash {listing_hash}') + if event_hash == listing_hash: + log.info(f'Listing {listing_hash} has been listed as a candidate in protocol') + voteBy = evt['args']['voteBy'] + if voteBy > int(time.time()): + owner = evt['args']['owner'] + return owner + + deployed = Protocol() From 76fd57f8e0ddd340c0be27728c76daa3acbda974 Mon Sep 17 00:00:00 2001 From: Will Button Date: Sun, 11 Aug 2019 16:26:15 -0700 Subject: [PATCH 5/6] Candidate already validated --- protocol/__init__.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/protocol/__init__.py b/protocol/__init__.py index 9707127..d452235 100644 --- a/protocol/__init__.py +++ b/protocol/__init__.py @@ -148,13 +148,9 @@ def send_data_hash(self, listing, data_hash): """ is_candidate = call(self.voting.is_candidate(listing)) candidate_is = call(self.voting.candidate_is(listing, constants.PROTOCOL_APPLICATION)) - if self.get_backend_address(): - if is_candidate and candidate_is: - receipt = send(self.w3, self.datatrust_key, self.datatrust.set_data_hash(listing, data_hash)) - return receipt - else: - log.critical(constants.INVALID_CANDIDATE_OR_POLL_CLOSED) - raise ValueError(constants.INVALID_CANDIDATE_OR_POLL_CLOSED) + if self.get_backend_address(): + receipt = send(self.w3, self.datatrust_key, self.datatrust.set_data_hash(listing, data_hash)) + return receipt else: log.critical(constants.NOT_DATATRUST) raise ValueError(constants.NOT_DATATRUST) From 318296ae22cbbc6fe038317149ede98f1ffce2ae Mon Sep 17 00:00:00 2001 From: Will Button Date: Sun, 11 Aug 2019 16:27:24 -0700 Subject: [PATCH 6/6] Candidate already validated --- protocol/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/protocol/__init__.py b/protocol/__init__.py index d452235..8ca9c13 100644 --- a/protocol/__init__.py +++ b/protocol/__init__.py @@ -146,8 +146,6 @@ def send_data_hash(self, listing, data_hash): """ On a successful post to the API db, send the data hash to protocol """ - is_candidate = call(self.voting.is_candidate(listing)) - candidate_is = call(self.voting.candidate_is(listing, constants.PROTOCOL_APPLICATION)) if self.get_backend_address(): receipt = send(self.w3, self.datatrust_key, self.datatrust.set_data_hash(listing, data_hash)) return receipt