crystal-lang · Feb 9, 2017 · Feb 9, 2017 · Feb 9, 2017
diff --git a/spec/std/file_spec.cr b/spec/std/file_spec.cr
@@ -944,4 +944,31 @@ describe "File" do
       expect_raises(IO::Error, "closed stream") { io.write_byte('a'.ord.to_u8) }
     end
   end
+
+  describe "utime" do
+    it "sets times with utime" do
+      filename = "#{__DIR__}/data/temp_write.txt"
+      File.write(filename, "")
+
+      atime = Time.new(2000, 1, 2)
+      mtime = Time.new(2000, 3, 4)
+
+      File.utime(atime, mtime, filename)
+
+      stat = File.stat(filename)
+      stat.atime.should eq(atime)
+      stat.mtime.should eq(mtime)
+
+      File.delete filename
+    end
+
+    it "raises if file not found" do
+      atime = Time.new(2000, 1, 2)
+      mtime = Time.new(2000, 3, 4)
+
+      expect_raises Errno, "Error setting time to file" do
+        File.utime(atime, mtime, "#{__DIR__}/nonexistent_file")
+      end
+    end
+  end
 end
diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr
@@ -134,8 +134,10 @@ module Crystal
       source = [source] unless source.is_a?(Array)
       program = new_program(source)
       node = parse program, source
-      node = program.semantic node, @stats, cleanup: !no_cleanup?
-      codegen program, node, source, output_filename unless @no_codegen
+      node = program.semantic node, cleanup: !no_cleanup?
+      result = codegen program, node, source, output_filename unless @no_codegen
+      print_macro_run_stats(program)
+      print_codegen_stats(result)
       Result.new program, node
     end
 
@@ -153,7 +155,8 @@ module Crystal
       source = [source] unless source.is_a?(Array)
       program = new_program(source)
       node = parse program, source
-      node, processor = program.top_level_semantic(node, @stats)
+      node, processor = program.top_level_semantic(node)
+      print_macro_run_stats(program)
       Result.new program, node
     end
 
@@ -169,6 +172,7 @@ module Crystal
       program.color = color?
       program.stdout = stdout
       program.show_error_trace = show_error_trace?
+      program.wants_stats = @stats
       program
     end
 
@@ -239,10 +243,12 @@ module Crystal
       if @cross_compile
         cross_compile program, units, lib_flags, output_filename
       else
-        codegen program, units, lib_flags, output_filename, output_dir
+        result = codegen program, units, lib_flags, output_filename, output_dir
       end
 
       CacheDir.instance.cleanup if @cleanup
+
+      result
     end
 
     private def cross_compile(program, units, lib_flags, output_filename)
@@ -278,16 +284,6 @@ module Crystal
         end
       end
 
-      if @stats
-        if units.size == reused
-          puts "Codegen (bc+obj): all previous .o files were reused"
-        elsif reused == 0
-          puts "Codegen (bc+obj): no previous .o files were reused"
-        else
-          puts "Codegen (bc+obj): #{reused}/#{units.size} .o files were reused"
-        end
-      end
-
       # We check again because maybe this directory was created in between (maybe with a macro run)
       if Dir.exists?(output_filename)
         error "can't use `#{output_filename}` as output filename because it's a directory"
@@ -300,6 +296,8 @@ module Crystal
           system %(#{CC} -o "#{output_filename}" "${@}" #{@link_flags} #{lib_flags}), object_names
         end
       end
+
+      {units.size, reused}
     end
 
     private def codegen_many_units(program, units, target_triple)
@@ -350,6 +348,41 @@ module Crystal
       unit.compile
     end
 
+    private def print_macro_run_stats(program)
+      return unless @stats && !program.compiled_macros_cache.empty?
+
+      puts
+      puts "Macro runs:"
+      program.compiled_macros_cache.each do |filename, compiled_macro_run|
+        print " - "
+        print filename
+        print ": "
+        if compiled_macro_run.elapsed == Time::Span.zero
+          print "reused previous compilation"
+        else
+          print compiled_macro_run.elapsed
+        end
+        puts
+      end
+    end
+
+    private def print_codegen_stats(result)
+      return unless @stats
+      return unless result
+
+      units_size, reused = result
+
+      puts
+      puts "Codegen (bc+obj):"
+      if units_size == reused
+        puts " - all previous .o files were reused"
+      elsif reused == 0
+        puts " - no previous .o files were reused"
+      else
+        puts " - #{reused}/#{units_size} .o files were reused"
+      end
+    end
+
     protected def target_machine
       @target_machine ||= begin
         triple = @target_triple || LLVM.default_target_triple

diff --git a/src/compiler/crystal/crystal_path.cr b/src/compiler/crystal/crystal_path.cr
@@ -2,6 +2,9 @@ require "./config"
 
 module Crystal
   struct CrystalPath
+    class Error < Exception
+    end
+
     def self.default_path
       ENV["CRYSTAL_PATH"]? || Crystal::Config.path
     end
@@ -40,7 +43,7 @@ module Crystal
       end
     end
 
-    def find(filename, relative_to = nil)
+    def find(filename, relative_to = nil) : Array(String)?
       relative_to = File.dirname(relative_to) if relative_to.is_a?(String)
       if filename.starts_with? '.'
         result = find_in_path_relative_to_dir(filename, relative_to)
@@ -157,9 +160,9 @@ module Crystal
       end
 
       if relative_to
-        raise "can't find file '#{filename}' relative to '#{relative_to}'"
+        raise Error.new("can't find file '#{filename}' relative to '#{relative_to}'")
       else
-        raise "can't find file '#{filename}'"
+        raise Error.new("can't find file '#{filename}'")
       end
     end
   end

diff --git a/src/compiler/crystal/macros.cr b/src/compiler/crystal/macros.cr
@@ -64,23 +64,30 @@ module Crystal::Macros
   # A simple example:
   #
   # ```
-  # # fetch.cr
-  # require "http/client"
-  #
-  # puts HTTP::Client.get(ARGV[0]).body
+  # # read.cr
+  # puts File.read(ARGV[0])
   # ```
   #
   # ```
   # # main.cr
-  # macro invoke_fetch
-  #   {{ run("./fetch", "http://example.com").stringify }}
+  # macro read_file_at_compile_time(filename)
+  #   {{ run("./read", filename).stringify }}
   # end
   #
-  # puts invoke_fetch
+  # puts read_file_at_compile_time("some_file.txt")
   # ```
   #
-  # The above generates a program that will have the contents of `http://example.com`.
-  # A connection to `http://example.com` is never made at runtime.
+  # The above generates a program that will have the contents of `some_file.txt`.
+  # The file, however, is read at compile time and will not be needed at runtime.
+  #
+  # NOTE: the compiler is allowed to cache the executable generated for
+  # *filename* and only recompile it if any of the files it depends on changes
+  # (their modified time). This is why it's **strongly discouraged** to use a program
+  # for `run` that changes in subsequent compilations (for example, if it executes
+  # shell commands at compile time, or other macro run programs). It's also strongly
+  # discouraged to have a macro run program take a lot of time, because this will
+  # slow down compilation times. Reading files is OK, opening an HTTP connection
+  # at compile-time will most likely result if very slow compilations.
   def run(filename, *args) : MacroId
   end
 

diff --git a/src/compiler/crystal/macros/macros.cr b/src/compiler/crystal/macros/macros.cr
@@ -16,8 +16,12 @@ class Crystal::Program
 
   # A cache of compiled "macro run" files.
   # The keys are filenames that were compiled, the values are  executable
-  # filenames ready to be run (so they don't need to be compiled twice)
-  @compiled_macros_cache = {} of String => String
+  # filenames ready to be run (so they don't need to be compiled twice),
+  # together with the time it took to compile them (it could be zero
+  # if we could reuse a previous compilation).
+  # The elapsed time is only needed for stats.
+  record CompiledMacroRun, filename : String, elapsed : Time::Span
+  property compiled_macros_cache = {} of String => CompiledMacroRun
 
   # Returns a new temporary file, which tries to be stored in the
   # cache directory associated to a program. This file is then added
@@ -76,7 +80,8 @@ class Crystal::Program
   end
 
   def macro_run(filename, args)
-    compiled_file = @compiled_macros_cache[filename] ||= macro_compile(filename)
+    compiled_macro_run = @compiled_macros_cache[filename] ||= macro_compile(filename)
+    compiled_file = compiled_macro_run.filename
 
     command = String.build do |str|
       str << compiled_file.inspect
@@ -90,9 +95,37 @@ class Crystal::Program
     {$?.success?, result}
   end
 
+  record RequireWithTimestamp, filename : String, epoch : Int64 do
+    JSON.mapping(filename: String, epoch: Int64)
+  end
+
   def macro_compile(filename)
+    time = wants_stats? ? Time.now : Time.epoch(0)
+
     source = File.read(filename)
 
+    # We store the executable relative to the cache directory for 'filename',
+    # that way if it's already there from a previous compilation, and no file
+    # that this program uses changes, we can simply avoid recompiling it again
+    #
+    # Note: it could happen that a macro run program runs macros that could
+    # change the program behaviour even if files don't change, but this is
+    # discouraged (and we should strongly document it) because it prevents
+    # incremental compiles.
+    program_dir = CacheDir.instance.directory_for(filename)
+    executable_path = File.join(program_dir, "macro_run")
+    recorded_requires_path = File.join(program_dir, "recorded_requires")
+    requires_path = File.join(program_dir, "requires")
+
+    # First, update times for the program dir, so it remains in the cache longer
+    # (this is specially useful if a macro run program is used by multiple programs)
+    now = Time.now
+    File.utime(now, now, program_dir)
+
+    if can_reuse_previous_compilation?(filename, executable_path, recorded_requires_path, requires_path)
+      return CompiledMacroRun.new(executable_path, Time::Span.zero)
+    end
+
     compiler = Compiler.new
 
     # Although release takes longer, once the bc is cached in .crystal
@@ -108,9 +141,76 @@ class Crystal::Program
     # No need to generate debug info for macro run programs
     compiler.debug = Crystal::Debug::None
 
-    safe_filename = filename.gsub(/[^a-zA-Z\_\-\.]/, "_")
-    tempfile_path = @program.new_tempfile("macro-run-#{safe_filename}")
-    compiler.compile Compiler::Source.new(filename, source), tempfile_path
-    tempfile_path
+    result = compiler.compile Compiler::Source.new(filename, source), executable_path
+
+    # Write the new files from which 'filename' depends into the cache dir
+    # (here we store how to obtain these files, because a require might use
+    # '/*' or '/**' and we need to recompile if a file is added or removed)
+    File.open(recorded_requires_path, "w") do |file|
+      result.program.recorded_requires.to_json(file)
+    end
+
+    # Together with their timestamp
+    # (this is the list of all effective files that were required)
+    requires_with_timestamps = result.program.requires.map do |required_file|
+      epoch = File.stat(required_file).mtime.epoch
+      RequireWithTimestamp.new(required_file, epoch)
+    end
+
+    File.open(requires_path, "w") do |file|
+      requires_with_timestamps.to_json(file)
+    end
+
+    elapsed_time = wants_stats? ? (Time.now - time) : Time::Span.zero
+
+    CompiledMacroRun.new(executable_path, elapsed_time)
+  end
+
+  private def can_reuse_previous_compilation?(filename, executable_path, recorded_requires_path, requires_path)
+    return false unless File.exists?(executable_path)
+    return false unless File.exists?(recorded_requires_path)
+    return false unless File.exists?(requires_path)
+
+    recorded_requires =
+      begin
+        Array(Program::RecordedRequire).from_json(File.read(recorded_requires_path))
+      rescue JSON::Error
+        return false
+      end
+
+    requires_with_timestamps =
+      begin
+        Array(RequireWithTimestamp).from_json(File.read(requires_path))
+      rescue JSON::Error
+        return false
+      end
+
+    # From the recorded requires we reconstruct the effective required files.
+    # We start with the target filename
+    required_files = Set{filename}
+    recorded_requires.map do |recorded_require|
+      begin
+        files = @program.find_in_path(recorded_require.filename, recorded_require.relative_to)
+        required_files.merge!(files) if files
+      rescue Crystal::CrystalPath::Error
+        # Maybe the file is gone
+        next
+      end
+    end
+
+    new_requires_with_timestamps = required_files.map do |required_file|
+      epoch = File.stat(required_file).mtime.epoch
+      RequireWithTimestamp.new(required_file, epoch)
+    end
+
+    # Quick check: if there are a different number of files, something changed
+    if requires_with_timestamps.size != new_requires_with_timestamps.size
+      return false
+    end
+
+    # Sort both requires and check if they are the same
+    requires_with_timestamps.sort_by! &.filename
+    new_requires_with_timestamps.sort_by! &.filename
+    requires_with_timestamps == new_requires_with_timestamps
   end
 end
diff --git a/src/compiler/crystal/program.cr b/src/compiler/crystal/program.cr
@@ -1,4 +1,5 @@
 require "llvm"
+require "json"
 require "./types"
 
 module Crystal
@@ -109,6 +110,9 @@ module Crystal
     # The main filename of this program
     property filename : String?
 
+    # If `true`, prints time and memory stats to `stdout`.
+    property? wants_stats = false
+
     def initialize
       super(self, self, "main")
 
@@ -406,9 +410,19 @@ module Crystal
       end
     end
 
+    record RecordedRequire, filename : String, relative_to : String? do
+      JSON.mapping(filename: String, relative_to: String?)
+    end
+    property recorded_requires = [] of RecordedRequire
+
+    # Rmembers that the program depends on this require.
+    def record_require(filename, relative_to) : Nil
+      recorded_requires << RecordedRequire.new(filename, relative_to)
+    end
+
     # Finds *filename* in the configured CRYSTAL_PATH for this program,
     # relative to *relative_to*.
-    def find_in_path(filename, relative_to = nil)
+    def find_in_path(filename, relative_to = nil) : Array(String)?
       crystal_path.find filename, relative_to
     end