Merge Karnov Group's excel-to-code with upstream excel_to_code

bin/excel_to_c

@@ -6,16 +6,16 @@ command = ExcelToC.new
 
 opts = OptionParser.new do |opts|
   CommonCommandLineOptions.set(command: command, options: opts, generates: "C", extension: "c")
-  
+
   opts.on('-c','--compile',"Compile the C") do
     command.actually_compile_code = true
   end
-  
+
   opts.on('--[no-]makefile', 'Generate a makefile. Default: no.') do |b|
     command.create_makefile = b
   end
 
-  
+
   opts.on('--[no-]rakefile', 'Generate a rakefile. Default: yes.') do |b|
     command.create_rakefile = b
   end

src/commands/common_command_line_options.rb

@@ -45,6 +45,7 @@ def self.set(command:, options:, generates:, extension:)
 
     options.on('-d', '--debug', "Fewer optimisations; the #{generates} should be more similar to the original Excel.") do
       command.should_inline_formulae_that_are_only_used_once = false
+      command.replace_reference_to_blanks_with_zeros = false
       command.extract_repeated_parts_of_formulae = false
     end
 

src/commands/excel_to_c.rb

@@ -45,6 +45,7 @@ def write_out_excel_as_code
     o.puts "// #{excel_file} approximately translated into C"
     o.puts "// definitions"
     o.puts "#define NUMBER_OF_REFS #{number_of_refs}"
+    o.puts "#define NUMBER_OF_RECURSION_PREVENT_VARS #{number_of_refs}" # This is only known after creating CompileToC, number_of_refs overapproximate it
     o.puts "#define EXCEL_FILENAME  #{excel_file.inspect}"
     o.puts "// end of definitions"
     o.puts
@@ -86,6 +87,7 @@ def write_out_excel_as_code
     c.settable = settable
     c.gettable = gettable
     c.rewrite(@formulae, @worksheet_c_names, o)
+    c.reset_sheets(@worksheet_c_names, o)
 
     # Output the named references
 
@@ -193,7 +195,15 @@ def write_fuby_ffi_interface
 
     name = output_name.downcase
     o = output("#{name}.rb")
-
+
+    sheet_resets = []
+    worksheets do |sheet_name, _|
+      sheet_name = c_name_for_worksheet_name(sheet_name)
+      sheet_resets.push("def reset_#{sheet_name}")
+      sheet_resets.push("  C.reset_#{sheet_name}")
+      sheet_resets.push("end\n")
+    end
+
     code = <<END
 require 'ffi'
 require 'singleton'
@@ -209,6 +219,8 @@ def reset
     C.reset
   end
 
+  #{sheet_resets.join("\n")}
+
   def method_missing(name, *arguments)
     if arguments.size == 0
       get(name)
@@ -220,7 +232,7 @@ def method_missing(name, *arguments)
   end
 
   def get(name)
-    return 0 unless C.respond_to?(name)
+    return nil unless C.respond_to?(name)
     ruby_value_from_excel_value(C.send(name))
   end
 
@@ -314,6 +326,10 @@ class ExcelValue < FFI::Struct
     o.puts "    # use this function to reset all cell values"
     o.puts "    attach_function 'reset', [], :void"
 
+    worksheets do |sheet_name, _|
+      sheet_name = c_name_for_worksheet_name(sheet_name)
+      o.puts "    attach_function 'reset_#{sheet_name}', [], :void"
+    end
 
     worksheets do |name, xml_filename|
       c_name = c_name_for_worksheet_name(name)

src/commands/excel_to_x.rb

@@ -109,6 +109,11 @@ class ExcelToX
   #   * false - the compiler leaves calculations in their original cells expanded. This may make debugging easier
   attr_accessor :should_inline_formulae_that_are_only_used_once
 
+  # Optional attribute, Boolean.
+  #   * true (default) - the compiler replaces references to blanks with zeros. This should increase performance
+  #   * false - the compiler leaves references to blanks. This make setable cells more predicatable and easing debugging.
+  attr_accessor :replace_reference_to_blanks_with_zeros
+
   # Optional attribute, Boolean.
   #   * true (default) - the compiler attempts to extract bits of calculation that appear in more than one formula into separate methods. This should increase performance
   #   * false - the compiler leaves calculations fully expanded. This may make debugging easier
@@ -267,6 +272,7 @@ def set_defaults
     # Setting this to false may make it easier to figure out errors
     self.extract_repeated_parts_of_formulae = true if @extract_repeated_parts_of_formulae == nil
     self.should_inline_formulae_that_are_only_used_once = true if @should_inline_formulae_that_are_only_used_once == nil
+    self.replace_reference_to_blanks_with_zeros = true if @should_inline_formulae_that_are_only_used_once == nil
 
     # This setting is used for debugging, and makes the system only do the conversion on a subset of the worksheets
     if self.isolate
@@ -754,7 +760,7 @@ def rewrite_row_and_column_references
   # them so every cell has its own definition
   def rewrite_shared_formulae_into_normal_formulae
     log.info "Rewriting shared formulae"
-    @formulae_shared = RewriteSharedFormulae.rewrite( @formulae_shared, @formulae_shared_targets)
+    @formulae_shared = RewriteSharedFormulae.rewrite( @formulae_shared, @formulae_shared_targets, @named_references)
     @shared_formulae_targets = :no_longer_needed # Allow the targets to be garbage collected.
   end
 
@@ -974,7 +980,7 @@ def simplify(cells = @formulae)
     @replace_arrays_with_single_cells_replacer ||= ReplaceArraysWithSingleCellsAst.new
     @replace_string_joins_on_ranges_replacer ||= ReplaceStringJoinOnRangesAST.new
     @sheetless_cell_reference_replacer ||= RewriteCellReferencesToIncludeSheetAst.new
-    @replace_references_to_blanks_with_zeros ||= ReplaceReferencesToBlanksWithZeros.new(@formulae, nil, inline_ast_decision)
+    @replace_references_to_blanks_with_zeros ||= ReplaceReferencesToBlanksWithZeros.new(@formulae, nil, inline_ast_decision, @named_references)
     @fix_subtotal_of_subtotals ||= FixSubtotalOfSubtotals.new(@formulae)
     # FIXME: Bodge to put it here as well, but seems to be required
     column_and_row_function_replacement = ReplaceColumnAndRowFunctionsAST.new
@@ -1012,8 +1018,12 @@ def simplify(cells = @formulae)
         @replace_arithmetic_on_ranges_replacer.map(ast)
         @replace_string_joins_on_ranges_replacer.map(ast)
         @wrap_formulae_that_return_arrays_replacer.map(ast)
-        @replace_references_to_blanks_with_zeros.current_sheet_name = ref.first
-        @replace_references_to_blanks_with_zeros.map(ast)
+        column_and_row_function_replacement.current_reference = ref.last
+        column_and_row_function_replacement.replace(ast)
+        if replace_reference_to_blanks_with_zeros
+          @replace_references_to_blanks_with_zeros.current_sheet_name = ref.first
+          @replace_references_to_blanks_with_zeros.map(ast)
+        end
         @fix_subtotal_of_subtotals.map(ast)
       rescue  Exception => e
         log.fatal "Exception when simplifying #{ref}: #{ast}"
@@ -1072,6 +1082,7 @@ def replace_formulae_with_their_results
     inline_replacer = InlineFormulaeAst.new
     inline_replacer.references = @formulae
     inline_replacer.inline_ast = inline_ast_decision
+    inline_replacer.named_references = @named_references
 
     value_replacer = MapFormulaeToValues.new
     value_replacer.original_excel_filename = excel_file
@@ -1111,9 +1122,11 @@ def replace_formulae_with_their_results
           if cell_address_replacement.replace(ast)
             references_that_need_updating[ref] = ast
           end
-          # FIXME: Shouldn't need to wrap ref.fist in an array
-          inline_replacer.current_sheet_name = [ref.first]
-          inline_replacer.map(ast)
+          if should_inline_formulae_that_are_only_used_once
+            # FIXME: Shouldn't need to wrap ref.fist in an array
+            inline_replacer.current_sheet_name = [ref.first]
+            inline_replacer.map(ast)
+          end
           # If a formula references a cell containing a value, the reference is replaced with the value (e.g., if A1 := 2 and A2 := A1 + 1 then becomes: A2 := 2 + 1)
           #require 'pry'; binding.pry if ref == [:"Outputs - Summary table", :E77]
           value_replacer.map(ast)
@@ -1127,9 +1140,11 @@ def replace_formulae_with_their_results
         end
       end
 
-      @named_references.each do |ref, ast|
-        inline_replacer.current_sheet_name = ref.is_a?(Array) ? [ref.first] : []
-        inline_replacer.map(ast)
+      if should_inline_formulae_that_are_only_used_once
+        @named_references.each do |ref, ast|
+          inline_replacer.current_sheet_name = ref.is_a?(Array) ? [ref.first] : []
+          inline_replacer.map(ast)
+        end
       end
 
       simplify(references_that_need_updating)
@@ -1328,6 +1343,16 @@ def ensure_there_is_a_good_set_of_cells_that_can_be_set_at_runtime
     @cells_that_can_be_set_at_runtime = cells_with_settable_values
   end
 
+  def make_blank_referenced_cells_settable(ref, ast)
+    unless replace_reference_to_blanks_with_zeros
+      if ast == nil
+        ast = [:number, 0.0]
+        @formulae[ref] = ast
+      end
+    end
+    ast
+  end
+
   # Returns a list of cells that are:
   # 1. Simple values (e.g., a string or a number)
   # 2. That are referenced in other formulae
@@ -1337,14 +1362,14 @@ def cells_with_settable_values
     log.info "Generating a good set of cells that should be settable"
 
     counter = CountFormulaReferences.new
-    count = counter.count(@formulae)
+    countList = counter.count(@formulae)
     settable_cells = {}
     settable_types = [:blank,:number,:null,:string,:shared_string,:constant,:percentage,:error,:boolean_true,:boolean_false]
 
-    count.each do |ref,count|
+    countList.each do |ref,count|
       next unless count >= 1 # No point making a cell that isn't reference settable
-      ast = @formulae[ref]
-      next unless ast # Sometimes empty cells are referenced. 
+      ast = make_blank_referenced_cells_settable(ref, @formulae[ref])
+      next unless ast # Sometimes empty cells are referenced.
       next unless settable_types.include?(ast.first)
       settable_cells[ref.first] ||= []
       settable_cells[ref.first] << ref.last.upcase

src/compile/c/compile_to_c.rb

@@ -1,20 +1,30 @@
 require_relative 'map_formulae_to_c'
+require 'set'
 
 class CompileToC
-  
+
   attr_accessor :settable
   attr_accessor :gettable
   attr_accessor :variable_set_counter
+  attr_accessor :variable_set_sheet_hash
+  attr_accessor :recursion_prevention_sheet_hash
   attr_accessor :allow_unknown_functions
-  
+
   def self.rewrite(*args)
     self.new.rewrite(*args)
   end
-
+
+  def init_sheet_hash(sheet_names)
+    Hash[sheet_names.map {|sheet_name| [sheet_name, Set.new]}]
+  end
+
   def rewrite(formulae, sheet_names, output)
     self.settable ||= lambda { |ref| false }
     self.gettable ||= lambda { |ref| true }
     @variable_set_counter ||= 0
+    @recursion_prevention_counter ||= 0
+    @variable_set_sheet_hash ||= init_sheet_hash(sheet_names.values.uniq)
+    @recursion_prevention_sheet_hash ||= init_sheet_hash(sheet_names.values.uniq)
 
     mapper = MapFormulaeToC.new
     mapper.allow_unknown_functions = self.allow_unknown_functions
@@ -27,7 +37,7 @@ def rewrite(formulae, sheet_names, output)
         worksheet_c_name = mapper.sheet_names[worksheet.to_s] || worksheet.to_s
         calculation = mapper.map(ast)
         name = worksheet_c_name.length > 0 ? "#{worksheet_c_name}_#{cell.downcase}" : cell.downcase
-        
+
         # Declare function as static so it can be inlined where possible
         static_or_not = gettable.call(ref) ? "" : "static "
 
@@ -54,17 +64,27 @@ def rewrite(formulae, sheet_names, output)
             output.puts "#{static_or_not}ExcelValue #{name}() {"
             output.puts "  static ExcelValue result;"
             output.puts "  if(variable_set[#{@variable_set_counter}] == 1) { return result;}"
+            output.puts "  if(recursion_prevention[#{@recursion_prevention_counter}] == 1) { return result;}"
+            output.puts "  recursion_prevention[#{@recursion_prevention_counter}] = 1;"
+
             mapper.initializers.each do |i|
-              output.puts "  #{i}"
+              output.puts "  #{i}".gsub("#{worksheet_c_name}_#{cell.downcase}()", "ZERO")
             end
+
             output.puts "  result = #{calculation};"
             output.puts "  variable_set[#{@variable_set_counter}] = 1;"
+            output.puts "  recursion_prevention[#{@recursion_prevention_counter}] = 0;"
             output.puts "  return result;"
             output.puts "}"
             output.puts
           end
         end
-        @variable_set_counter += 1
+        unless worksheet_c_name.empty?
+          @variable_set_sheet_hash[worksheet_c_name].add(@variable_set_counter)
+          @recursion_prevention_sheet_hash[worksheet_c_name].add(@variable_set_counter)
+          @variable_set_counter += 1
+          @recursion_prevention_counter += 1
+        end
         mapper.reset
       rescue Exception => e
         puts "Exception at #{ref} #{ast}"
@@ -75,8 +95,24 @@ def rewrite(formulae, sheet_names, output)
           end
         end
         raise
-      end      
+      end
+    end
+  end
+
+  def reset_sheets(sheet_names, output)
+    sheet_names = sheet_names.values.uniq
+    sheet_names.each do |sheet_name|
+      output.puts "void reset_#{sheet_name}()\n{"
+
+      @variable_set_sheet_hash[sheet_name].each do |variable|
+        output.puts "  variable_set[#{variable}] = 0;"
+      end
+
+      @recursion_prevention_sheet_hash[sheet_name].each do |variable|
+        output.puts "  recursion_prevention[#{variable}] = 0;"
+      end
+
+      output.puts "}\n"
     end
   end
-
 end

src/compile/c/excel_to_c_runtime.c

@@ -166,11 +166,13 @@ static void free_all_allocated_memory() {
 }
 
 static int variable_set[NUMBER_OF_REFS];
+static int recursion_prevention[NUMBER_OF_RECURSION_PREVENT_VARS];
 
 // Resets all cached and malloc'd values
 void reset() {
   free_all_allocated_memory();
   memset(variable_set, 0, sizeof(variable_set));
+  memset(recursion_prevention, 0, sizeof(recursion_prevention));
 }
 
 // Handy macros
@@ -3017,11 +3019,18 @@ static ExcelValue hlookup(ExcelValue lookup_value_v,ExcelValue lookup_table_v, E
     for(i=0; i< columns; i++) {
       possible_match_v = array[i];
       if(lookup_value_v.type != possible_match_v.type) continue;
-      if(more_than(possible_match_v,lookup_value_v).number == true) {
-        if(i == 0) return NA;
-        return array[((((int) row_number_v.number)-1)*columns)+(i-1)];
-      } else {
-        last_good_match = i;
+      if(lookup_value_v.type == ExcelString && possible_match_v.type == ExcelString)
+      {
+        if (strstr(possible_match_v.string, lookup_value_v.string) != NULL)
+          last_good_match = i;
+      } else
+      {
+        if(more_than(possible_match_v,lookup_value_v).number == true) {
+          if(i == 0) return NA;
+          return array[((((int) row_number_v.number)-1)*columns)+(i-1)];
+        } else {
+          last_good_match = i;
+        }
       }
     }
     return array[((((int) row_number_v.number)-1)*columns)+(last_good_match)];

src/rewrite/ast_copy_formula.rb

@@ -4,10 +4,12 @@
 class AstCopyFormula
   attr_accessor :rows_to_move
   attr_accessor :columns_to_move
+  attr_accessor :named_references
 
-  def initialize
+  def initialize(named_references = {})
     self.rows_to_move = 0
     self.columns_to_move = 0
+    self.named_references = named_references
   end
 
   DO_NOT_MAP = {:number => true, :string => true, :blank => true, :null => true, :error => true, :boolean_true => true, :boolean_false => true, :operator => true, :comparator => true}
@@ -26,6 +28,9 @@ def copy(ast)
 
   def cell(reference)
     r = Reference.for(reference)
+    if self.named_references.key?(reference)
+      return [:cell, reference]
+    end
     [:cell,r.offset(rows_to_move,columns_to_move)]
   end