Merge pull request #68 from jondkinney/feature/support-unicode-chars

Allow unicode chars, force_encoding('UTF-8')

Author: elia

.github/workflows/main.yml

@@ -25,3 +25,5 @@ jobs:
         run: bin/setup
       - name: Run test
         run: "bundle exec rake"
+      - name: Run test
+        run: "env LANG=C LC_ALL=C bundle exec rake"

lib/erb/formatter.rb

@@ -29,20 +29,20 @@ class Error < StandardError; end
   SPACES = /\s+/m
 
   # https://stackoverflow.com/a/317081
-  ATTR_NAME = %r{[^\r\n\t\f\v= '"<>]*[^\r\n\t\f\v= '"<>/]} # not ending with a slash
-  UNQUOTED_VALUE = %r{[^<>'"\s]+}
-  UNQUOTED_ATTR = %r{#{ATTR_NAME}=#{UNQUOTED_VALUE}}
-  SINGLE_QUOTE_ATTR = %r{(?:#{ATTR_NAME}='[^']*?')}m
-  DOUBLE_QUOTE_ATTR = %r{(?:#{ATTR_NAME}="[^"]*?")}m
-  BAD_ATTR = %r{#{ATTR_NAME}=\s+}
+  ATTR_NAME = %r{[^\r\n\t\f\v= '"<>]*[^\r\n\t\f\v= '"<>/]}u # not ending with a slash
+  UNQUOTED_VALUE = %r{[^<>'"\s]+}u
+  UNQUOTED_ATTR = %r{#{ATTR_NAME}=#{UNQUOTED_VALUE}}u
+  SINGLE_QUOTE_ATTR = %r{(?:#{ATTR_NAME}='[^']*?')}mu
+  DOUBLE_QUOTE_ATTR = %r{(?:#{ATTR_NAME}="[^"]*?")}mu
+  BAD_ATTR = %r{#{ATTR_NAME}=\s+}u
   QUOTED_ATTR = Regexp.union(SINGLE_QUOTE_ATTR, DOUBLE_QUOTE_ATTR)
   ATTR = Regexp.union(SINGLE_QUOTE_ATTR, DOUBLE_QUOTE_ATTR, UNQUOTED_ATTR, UNQUOTED_VALUE)
   MULTILINE_ATTR_NAMES = %w[class data-action]
 
   ERB_TAG = %r{(<%(?:==|=|-|))\s*(.*?)\s*(-?%>)}m
   ERB_PLACEHOLDER = %r{erb[a-z0-9]+tag}
 
-  TAG_NAME = /[a-z0-9_:-]+/
+  TAG_NAME = /[a-z0-9_:-]+/u
   TAG_NAME_ONLY = /\A#{TAG_NAME}\z/
   HTML_ATTR = %r{\s+#{SINGLE_QUOTE_ATTR}|\s+#{DOUBLE_QUOTE_ATTR}|\s+#{UNQUOTED_ATTR}|\s+#{ATTR_NAME}}m
   HTML_TAG_OPEN = %r{<(#{TAG_NAME})((?:#{HTML_ATTR})*)(\s*?)(/>|>)}m
@@ -79,11 +79,14 @@ def self.format(source, filename: nil)
   end
 
   def initialize(source, line_width: 80, single_class_per_line: false, filename: nil, css_class_sorter: nil, debug: $DEBUG)
-    @original_source = source
+    @original_source = source.to_s
+    @original_source = +@original_source if @original_source.frozen?
+    @original_source.force_encoding('UTF-8')
+
     @filename = filename || '(erb)'
     @line_width = line_width
-    @source = remove_front_matter source.dup
-    @html = +""
+    @source = remove_front_matter @original_source.dup
+    @html = +"".force_encoding('UTF-8')
     @debug = debug
     @single_class_per_line = single_class_per_line
     @css_class_sorter = css_class_sorter

test/erb/test_formatter.rb

@@ -12,11 +12,11 @@ def test_simple_tag
   end
 
   def test_fixtures
-    Dir["#{__dir__}/../fixtures/*.html.erb"].each do |erb_path|
+    Dir["#{__dir__}/../fixtures/*.html.erb"].shuffle.each do |erb_path|
       expected_path = erb_path.chomp('.erb') + '.expected.erb'
 
       # File.write expected_path, ERB::Formatter.format(File.read(erb_path))
-      assert_equal(File.read(expected_path), ERB::Formatter.new(File.read(erb_path)).to_s, "Formatting of #{erb_path} failed")
+      assert_equal(File.read(expected_path, encoding: "UTF-8"), ERB::Formatter.new(File.read(erb_path)).to_s, "Formatting of #{erb_path} failed")
     end
   end
 

test/fixtures/utf8.html.erb

@@ -1,4 +1,4 @@
-<h3>Hi <%= @merchant.first_name %></h3>
+<h3>Hi <%= @merchant.first_name %> 🙌 ↯</h3>
 <h1>SOS! Deine PEND-Anbindung wurde SUSPENDIERT!</h1>
 <p>Die PEND-Anbindung von <%= @gateway.title %> auf deinem Shop <%= @shop.name %> wurde suspendiert!</p>
 
@@ -16,4 +16,7 @@
 <%= form_with model: model, url: path, method: :patch do |f| %>
   <%= f.select :type, [["не выбрано", nil], %w[Тип1 type1], %w[Тип2 type2], %w[Тип3 type3], %w[Тип4 type4], %w[Тип5 type5]], {}, { class: "select" } %>
   <%= f.submit "Сохранить" %>
+  The dash: more—effortlessly
+  The copyright: ©
+  The quote: Figma and Tailwind ” are used
 <% end %>

test/fixtures/utf8.html.expected.erb

@@ -1,5 +1,6 @@
 <h3>Hi
-  <%= @merchant.first_name %></h3>
+  <%= @merchant.first_name %>
+  🙌 ↯</h3>
 <h1>SOS! Deine PEND-Anbindung wurde SUSPENDIERT!</h1>
 <p>Die PEND-Anbindung von
   <%= @gateway.title %>
@@ -38,4 +39,6 @@
            {},
            { class: "select" } %>
   <%= f.submit "Сохранить" %>
+  The dash: more—effortlessly The copyright: © The quote: Figma and Tailwind ”
+  are used
 <% end %>