Core: erase all \newline before parsing

The \newline is treated as a line continuation. Handling \newline in
the hidden channel is not ideal as it might break tokens. For example,
" \newline " will emit two BLANK tokens, which is not what we want.
Now \newline is erased before parsing. The drawback is it might not be
right when handling single quoted string and comments.

2 files changed, 15 insertions, 8 deletions

diff --git a/src/core/bash_ast.cpp b/src/core/bash_ast.cpp
index 213264c..f3afc30 100644
--- a/src/core/bash_ast.cpp
+++ b/src/core/bash_ast.cpp
@@ -26,6 +26,7 @@
 #include <sstream>
 #include <thread>
 
+#include <boost/algorithm/string/erase.hpp>
 #include <boost/numeric/conversion/cast.hpp>
 
 #include "core/exceptions.h"
@@ -34,13 +35,19 @@
 #include "libbashParser.h"
 #include "libbashWalker.h"
 
-bash_ast::bash_ast(const std::istream& source,
-                   std::function<pANTLR3_BASE_TREE(plibbashParser)> p): parse(p)
+void bash_ast::read_script(const std::istream& source)
 {
   std::stringstream stream;
   stream << source.rdbuf();
   script = stream.str();
-  init_parser(script, "unknown source");
+  boost::algorithm::erase_all(script, "\\\n");
+}
+
+bash_ast::bash_ast(const std::istream& source,
+                   std::function<pANTLR3_BASE_TREE(plibbashParser)> p): parse(p)
+{
+  read_script(source);
+  init_parser("unknown source");
 }
 
 bash_ast::bash_ast(const std::string& script_path,
@@ -51,9 +58,8 @@ bash_ast::bash_ast(const std::string& script_path,
   if(!file_stream)
     throw libbash::parse_exception(script_path + " can't be read");
 
-  stream << file_stream.rdbuf();
-  script = stream.str();
-  init_parser(script, script_path);
+  read_script(file_stream);
+  init_parser(script_path);
 }
 
 namespace
@@ -78,7 +84,7 @@ namespace
   }
 }
 
-void bash_ast::init_parser(const std::string& script, const std::string& script_path)
+void bash_ast::init_parser(const std::string& script_path)
 {
   input.reset(antlr3NewAsciiStringInPlaceStream(
     reinterpret_cast<pANTLR3_UINT8>(const_cast<char*>(script.c_str())),
diff --git a/src/core/bash_ast.h b/src/core/bash_ast.h
index 8aa291a..fcf031c 100644
--- a/src/core/bash_ast.h
+++ b/src/core/bash_ast.h
@@ -67,7 +67,8 @@ class bash_ast: public boost::noncopyable
 
   typedef std::unique_ptr<libbashWalker_Ctx_struct, std::function<void(libbashWalker_Ctx_struct*)>> walker_pointer;
 
-  void init_parser(const std::string& script, const std::string& script_path);
+  void read_script(const std::istream& source);
+  void init_parser(const std::string& script_path);
   walker_pointer create_walker(interpreter& walker,
                                antlr_pointer<ANTLR3_COMMON_TREE_NODE_STREAM_struct>& nodes);