From 0642b6244a23bade095b036e36857f8f44d92fd5 Mon Sep 17 00:00:00 2001 From: Tom Kent Date: Fri, 4 Jan 2019 21:15:30 -0600 Subject: [PATCH 1/2] Output a dot every 100KB of FTP transfer This should help keep CircleCI from timing out large file transfers. --- reports/src/boost_wide_report.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/reports/src/boost_wide_report.py b/reports/src/boost_wide_report.py index 0d5b8b4..86390cb 100644 --- a/reports/src/boost_wide_report.py +++ b/reports/src/boost_wide_report.py @@ -284,6 +284,25 @@ def update( self ): utils.log( ' Skipping "%s" due to errors (%s)' % ( self.source_, msg ) ) +class FtpFileProgress( object ): + def __init__(self, filename, mode='wb'): + self.f = open( filename, mode ) + self.bytes_processed = 0 + self.last_output = 0 + self.output_size = 100000 + + def write(self, file_bytes): + self.f.write(file_bytes) + self.bytes_processed += len(file_bytes) + + if self.bytes_processed > self.last_output + self.output_size: + sys.stdout.write('.') + self.last_output += self.output_size + + def close(self): + self.f.close() + + def ftp_task( site, site_path , destination, filter_runners = None ): __log__ = 1 utils.log( '' ) @@ -303,7 +322,7 @@ def ftp_task( site, site_path , destination, filter_runners = None ): def synchronize(): for source in d[0]: utils.log( 'Copying "%s"' % source ) - result = open( os.path.join( destination, source ), 'wb' ) + result = FtpFileProgress( os.path.join( destination, source ), 'wb') f.retrbinary( 'RETR %s' % source, result.write ) result.close() mod_date = find_by_name( source_content, source ).date From 5afcd10263f5f7afa450860764c0feb4fd8bf71b Mon Sep 17 00:00:00 2001 From: Tom Kent Date: Sat, 5 Jan 2019 07:41:37 -0600 Subject: [PATCH 2/2] Only print a dot every 1M, 100KB is too much. --- reports/src/boost_wide_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reports/src/boost_wide_report.py b/reports/src/boost_wide_report.py index 86390cb..3d52cc3 100644 --- a/reports/src/boost_wide_report.py +++ b/reports/src/boost_wide_report.py @@ -289,7 +289,7 @@ def __init__(self, filename, mode='wb'): self.f = open( filename, mode ) self.bytes_processed = 0 self.last_output = 0 - self.output_size = 100000 + self.output_size = 1000000 def write(self, file_bytes): self.f.write(file_bytes)