Add line numbers support (#10)

araujogui · web-flow · commit 53ee4927c0d9 · 2026-02-13T15:25:17.000-05:00
* feat: add line numbers support

* build: add cstdint header

* refactor: remove variable underscore

* refactor: review
diff --git a/README.md b/README.md
@@ -6,6 +6,7 @@ A fast C++ lexer for extracting named exports from CommonJS modules. This librar
 
 - **Fast**: Zero-copy parsing for most exports using `std::string_view`
 - **Accurate**: Handles complex CommonJS patterns including re-exports, Object.defineProperty, and transpiler output
+- **Source Locations**: Each export includes a 1-based line number for tooling integration
 - **Unicode Support**: Properly unescapes JavaScript string literals including `\u{XXXX}` and surrogate pairs
 - **Optional SIMD Acceleration**: Can use [simdutf](https://github.com/simdutf/simdutf) for faster string operations
 - **No Dependencies**: Single-header distribution available (simdutf is optional)
@@ -49,20 +50,21 @@ int main() {
   if (result) {
     std::cout << "Exports found:" << std::endl;
     for (const auto& exp : result->exports) {
-      std::cout << "  - " << lexer::get_string_view(exp) << std::endl;
+      std::cout << "  - " << lexer::get_string_view(exp)
+                << " (line " << exp.line << ")" << std::endl;
     }
   }
-  
+
   return 0;
 }
 ```
 
 Output:
 ```
 Exports found:
-  - foo
-  - bar
-  - baz
+  - foo (line 2)
+  - bar (line 3)
+  - baz (line 4)
 ```
 
 ## API Reference
@@ -85,11 +87,22 @@ Parses CommonJS source code and extracts export information.
 
 ```cpp
 struct lexer_analysis {
-  std::vector<export_string> exports;      // Named exports
-  std::vector<export_string> re_exports;   // Re-exported module specifiers
+  std::vector<export_entry> exports;      // Named exports
+  std::vector<export_entry> re_exports;   // Re-exported module specifiers
+};
+```
+
+### `lexer::export_entry`
+
+```cpp
+struct export_entry {
+  export_string name;
+  uint32_t line;  // 1-based line number
 };
 ```
 
+Each export/re-export entry includes the name and the 1-based line number where it was found in the source.
+
 ### `lexer::export_string`
 
 ```cpp
@@ -104,9 +117,10 @@ Export names are stored as a variant to avoid unnecessary copies:
 
 ```cpp
 inline std::string_view get_string_view(const export_string& s);
+inline std::string_view get_string_view(const export_entry& e);
 ```
 
-Helper function to get a `string_view` from either variant type.
+Helper function to get a `string_view` from an `export_string` or `export_entry`.
 
 ### `lexer::get_last_error`
 
diff --git a/include/merve/parser.h b/include/merve/parser.h
@@ -3,6 +3,7 @@
 
 #include "merve/version.h"
 
+#include <cstdint>
 #include <optional>
 #include <string>
 #include <string_view>
@@ -47,6 +48,14 @@ enum lexer_error {
  */
 using export_string = std::variant<std::string, std::string_view>;
 
+/**
+ * @brief An export name together with its 1-based source line number.
+ */
+struct export_entry {
+  export_string name;
+  uint32_t line;  // 1-based line number
+};
+
 /**
  * @brief Result of parsing a CommonJS module.
  */
@@ -61,7 +70,7 @@ struct lexer_analysis {
    * - module.exports = { a, b, c }
    * - Object.defineProperty(exports, 'name', {...})
    */
-  std::vector<export_string> exports{};
+  std::vector<export_entry> exports{};
 
   /**
    * @brief Module specifiers from re-export patterns.
@@ -72,7 +81,7 @@ struct lexer_analysis {
    * - __export(require('other'))
    * - Object.keys(require('other')).forEach(...)
    */
-  std::vector<export_string> re_exports{};
+  std::vector<export_entry> re_exports{};
 };
 
 /**
@@ -89,6 +98,13 @@ inline std::string_view get_string_view(const export_string& s) {
   return std::visit([](const auto& v) -> std::string_view { return v; }, s);
 }
 
+/**
+ * @brief Get a string_view from an export_entry (delegates to the name field).
+ */
+inline std::string_view get_string_view(const export_entry& e) {
+  return get_string_view(e.name);
+}
+
 /**
  * @brief Parse CommonJS source code and extract export information.
  *
diff --git a/src/parser.cpp b/src/parser.cpp
@@ -325,6 +325,8 @@ class CJSLexer {
   uint16_t openTokenDepth;
   uint16_t templateDepth;
 
+  uint32_t line;
+
   bool lastSlashWasDivision;
   bool nextBraceIsClass;
 
@@ -335,8 +337,16 @@ class CJSLexer {
   StarExportBinding* starExportStack;
   const StarExportBinding* STAR_EXPORT_STACK_END;
 
-  std::vector<export_string>& exports;
-  std::vector<export_string>& re_exports;
+  std::vector<export_entry>& exports;
+  std::vector<export_entry>& re_exports;
+
+  // Increments `line` when consuming a line terminator.
+  // - Counts '\n' as a newline.
+  // - Counts '\r' as a newline only when it is not part of a CRLF sequence.
+  //   (i.e., the next character is not '\n' or we're at end-of-input.)
+  void countNewline(char ch) {
+    line += (ch == '\n') || (ch == '\r' && (pos + 1 >= end || *(pos + 1) != '\n'));
+  }
 
   // Character classification helpers using lookup tables
   static bool isBr(char c) {
@@ -495,6 +505,8 @@ class CJSLexer {
           return ch;
       } else if (!isBrOrWs(ch)) {
         return ch;
+      } else {
+        countNewline(ch);
       }
     } while (pos++ < end);
     return ch;
@@ -503,8 +515,10 @@ class CJSLexer {
   void lineComment() {
     while (pos++ < end) {
       char ch = *pos;
-      if (ch == '\n' || ch == '\r')
+      if (ch == '\n' || ch == '\r') {
+        countNewline(ch);
         return;
+      }
     }
   }
 
@@ -516,6 +530,7 @@ class CJSLexer {
         pos++;
         return;
       }
+      countNewline(ch);
     }
   }
 
@@ -527,8 +542,13 @@ class CJSLexer {
       if (ch == '\\') {
         if (pos + 1 >= end) break;
         ch = *++pos;
-        if (ch == '\r' && *(pos + 1) == '\n')
-          pos++;
+        if (ch == '\r') {
+          ++line;
+          if (*(pos + 1) == '\n')
+            pos++;
+        } else if (ch == '\n') {
+          ++line;
+        }
       } else if (isBr(ch))
         break;
     }
@@ -580,8 +600,12 @@ class CJSLexer {
       }
       if (ch == '`')
         return;
-      if (ch == '\\' && pos + 1 < end)
+      if (ch == '\\' && pos + 1 < end) {
         pos++;
+        countNewline(*pos);
+      } else {
+        countNewline(ch);
+      }
     }
     syntaxError(lexer_error::UNTERMINATED_TEMPLATE_STRING);
   }
@@ -614,7 +638,7 @@ class CJSLexer {
 #endif
   }
 
-  void addExport(std::string_view export_name) {
+  void addExport(std::string_view export_name, uint32_t at_line) {
     // Skip surrounding quotes if present
     if (!export_name.empty() && (export_name.front() == '\'' || export_name.front() == '"')) {
       export_name.remove_prefix(1);
@@ -625,11 +649,11 @@ class CJSLexer {
     if (!needsUnescaping(export_name)) {
       // Check if this export already exists (avoid duplicates)
       for (const auto& existing : exports) {
-        if (get_string_view(existing) == export_name) {
+        if (get_string_view(existing.name) == export_name) {
           return; // Already exists, skip
         }
       }
-      exports.push_back(export_name);
+      exports.push_back(export_entry{export_name, at_line});
       return;
     }
 
@@ -644,14 +668,14 @@ class CJSLexer {
 
     // Check if this export already exists (avoid duplicates)
     for (const auto& existing : exports) {
-      if (get_string_view(existing) == name) {
+      if (get_string_view(existing.name) == name) {
         return; // Already exists, skip
       }
     }
-    exports.push_back(std::move(unescaped.value()));
+    exports.push_back(export_entry{std::move(unescaped.value()), at_line});
   }
 
-  void addReexport(std::string_view reexport_name) {
+  void addReexport(std::string_view reexport_name, uint32_t at_line) {
     // Skip surrounding quotes if present
     if (!reexport_name.empty() && (reexport_name.front() == '\'' || reexport_name.front() == '"')) {
       reexport_name.remove_prefix(1);
@@ -660,7 +684,7 @@ class CJSLexer {
 
     // Fast path: no escaping needed, use string_view directly
     if (!needsUnescaping(reexport_name)) {
-      re_exports.push_back(reexport_name);
+      re_exports.push_back(export_entry{reexport_name, at_line});
       return;
     }
 
@@ -670,7 +694,7 @@ class CJSLexer {
       return;  // Skip invalid escape sequences
     }
 
-    re_exports.push_back(std::move(unescaped.value()));
+    re_exports.push_back(export_entry{std::move(unescaped.value()), at_line});
   }
 
   bool readExportsOrModuleDotExports(char ch) {
@@ -712,7 +736,7 @@ class CJSLexer {
           switch (requireType) {
             case RequireType::ExportStar:
             case RequireType::ExportAssign:
-              addReexport(std::string_view(reexportStart, reexportEnd - reexportStart));
+              addReexport(std::string_view(reexportStart, reexportEnd - reexportStart), line);
               return true;
             default:
               if (starExportStack < STAR_EXPORT_STACK_END) {
@@ -773,7 +797,7 @@ class CJSLexer {
             return;
           }
         }
-        addExport(std::string_view(startPos, endPos - startPos));
+        addExport(std::string_view(startPos, endPos - startPos), line);
       } else if (ch == '\'' || ch == '"') {
         const char* start = pos;
         stringLiteral(ch);
@@ -786,7 +810,7 @@ class CJSLexer {
             pos = revertPos;
             return;
           }
-          addExport(std::string_view(start, end_pos - start));
+          addExport(std::string_view(start, end_pos - start), line);
         }
       } else if (ch == '.' && matchesAt(pos + 1, end, "..")) {
         pos += 3;
@@ -825,7 +849,7 @@ class CJSLexer {
           const char* endPos = pos;
           ch = commentWhitespace();
           if (ch == '=') {
-            addExport(std::string_view(startPos, endPos - startPos));
+            addExport(std::string_view(startPos, endPos - startPos), line);
             return;
           }
         }
@@ -843,7 +867,7 @@ class CJSLexer {
           pos++;
           ch = commentWhitespace();
           if (ch != '=') break;
-          addExport(std::string_view(startPos, endPos - startPos));
+          addExport(std::string_view(startPos, endPos - startPos), line);
         }
         break;
       }
@@ -974,7 +998,7 @@ class CJSLexer {
             ch = commentWhitespace();
             if (ch != ':') break;
             if (exportStart && exportEnd)
-              addExport(std::string_view(exportStart, exportEnd - exportStart));
+              addExport(std::string_view(exportStart, exportEnd - exportStart), line);
             pos = revertPos;
             return;
           } else if (ch == 'g') {
@@ -1042,7 +1066,7 @@ class CJSLexer {
             ch = commentWhitespace();
             if (ch != ')') break;
             if (exportStart && exportEnd)
-              addExport(std::string_view(exportStart, exportEnd - exportStart));
+              addExport(std::string_view(exportStart, exportEnd - exportStart), line);
             return;
           }
           break;
@@ -1406,7 +1430,7 @@ class CJSLexer {
           StarExportBinding* curCheckBinding = &starExportStack_[0];
           while (curCheckBinding != starExportStack) {
             if (curCheckBinding->id == id) {
-              addReexport(curCheckBinding->specifier);
+              addReexport(curCheckBinding->specifier, line);
               pos = revertPos;
               return;
             }
@@ -1506,9 +1530,10 @@ class CJSLexer {
   }
 
 public:
-  CJSLexer(std::vector<export_string>& out_exports, std::vector<export_string>& out_re_exports)
+  CJSLexer(std::vector<export_entry>& out_exports, std::vector<export_entry>& out_re_exports)
     : source(nullptr), pos(nullptr), end(nullptr), lastTokenPos(nullptr),
       templateStackDepth(0), openTokenDepth(0), templateDepth(0),
+      line(1),
       lastSlashWasDivision(false), nextBraceIsClass(false),
       templateStack_{}, openTokenPosStack_{}, openClassPosStack{},
       starExportStack_{}, starExportStack(nullptr), STAR_EXPORT_STACK_END(nullptr),
@@ -1525,6 +1550,7 @@ class CJSLexer {
     templateStackDepth = 0;
     openTokenDepth = 0;
     templateDepth = std::numeric_limits<uint16_t>::max();
+    line = 1;
     lastSlashWasDivision = false;
     starExportStack = &starExportStack_[0];
     STAR_EXPORT_STACK_END = &starExportStack_[MAX_STAR_EXPORTS - 1];
@@ -1549,8 +1575,10 @@ class CJSLexer {
     while (pos++ < end) {
       ch = *pos;
 
-      if (ch == ' ' || (ch < 14 && ch > 8))
+      if (ch == ' ' || (ch < 14 && ch > 8)) {
+        countNewline(ch);
         continue;
+      }
 
       if (openTokenDepth == 0) {
         switch (ch) {
diff --git a/tests/real_world_tests.cpp b/tests/real_world_tests.cpp
