gccrs: proc_macro: Add from string implementation

Add a callback registration function into the proc macro library so the
compiler can register it's own lexing/parsing functions on load.

gcc/rust/ChangeLog:

	* expand/rust-proc-macro.cc (tokenstream_from_string): Add a
	function that creates a tokenstream from a given string.
	(load_macros_array): Add call to registration function.

libgrust/ChangeLog:

	* libproc_macro/proc_macro.cc (proc_macro_register_from_str):
	Add registration function.
	* libproc_macro/proc_macro.h (proc_macro_register_from_str):
	Add registration function prototype.
	* libproc_macro/tokenstream.cc (TokenStream::make_tokenstream):
	Add a new constructor from a string that uses the registered
	callback.
	(TokenStream__from_string): Add call to new constructor.
	* libproc_macro/tokenstream.h: Add registration
	declaration.
	* libproc_macro/registration.h: New file.

Signed-off-by: Pierre-Emmanuel Patry <pierre-emmanuel.patry@embecosm.com>

gcc/rust/expand/rust-proc-macro.cc

@@ -16,6 +16,8 @@
 
 #include "rust-diagnostics.h"
 #include "rust-proc-macro.h"
+#include "rust-lex.h"
+#include "rust-token-converter.h"
 #ifndef _WIN32
 #include <dlfcn.h>
 #endif
@@ -24,6 +26,60 @@ namespace Rust {
 
 const std::string PROC_MACRO_DECL_PREFIX = "__gccrs_proc_macro_decls_";
 
+ProcMacro::TokenStream
+tokenstream_from_string (std::string &data, bool &lex_error)
+{
+  // FIXME: Insert location pointing to call site in tokens
+  Lexer lex (data);
+
+  std::vector<const_TokenPtr> tokens;
+  TokenPtr ptr;
+  for (ptr = lex.build_token ();
+       ptr != nullptr && ptr->get_id () != END_OF_FILE;
+       ptr = lex.build_token ())
+    {
+      tokens.emplace_back (ptr);
+    }
+
+  if (ptr == nullptr)
+    {
+      lex_error = true;
+      return ProcMacro::TokenStream::make_tokenstream ();
+    }
+
+  lex_error = false;
+  return convert (tokens);
+}
+
+static_assert (
+  std::is_same<decltype (tokenstream_from_string) *,
+	       ProcMacro::from_str_function_t>::value,
+  "Registration callback signature not synced, check proc macro internals.");
+
+template <typename Symbol, typename Callback>
+bool
+register_callback (void *handle, Symbol, std::string symbol_name,
+		   Callback callback)
+{
+  void *addr = dlsym (handle, symbol_name.c_str ());
+  if (addr == nullptr)
+    {
+      rust_error_at (Location (),
+		     "Callback registration symbol (%s) missing from "
+		     "proc macro, wrong version?",
+		     symbol_name.c_str ());
+      return false;
+    }
+
+  auto storage = reinterpret_cast<Symbol *> (addr);
+  *storage = callback;
+
+  return true;
+}
+
+#define REGISTER_CALLBACK(HANDLE, SYMBOL, CALLBACK)                            \
+  register_callback (HANDLE, SYMBOL, #SYMBOL, CALLBACK)
+
 const ProcMacro::ProcmacroArray *
 load_macros_array (std::string path)
 {
@@ -36,6 +92,10 @@ load_macros_array (std::string path)
       return nullptr;
     }
 
+  if (!REGISTER_CALLBACK (handle, __gccrs_pm_callback_from_str_fn,
+			  tokenstream_from_string))
+    return nullptr;
+
   // FIXME: Add CrateStableId handling, right now all versions may be loaded,
   // even incompatible ones.
   return *reinterpret_cast<const ProcMacro::ProcmacroArray **> (
@@ -47,6 +107,8 @@ load_macros_array (std::string path)
 #endif
 }
 
+#undef REGISTER_CALLBACK
+
 const std::vector<ProcMacro::Procmacro>
 load_macros (std::string path)
 {

libgrust/libproc_macro/proc_macro.cc

@@ -50,3 +50,5 @@ Procmacro::make_bang (const char *name, BangMacro macro)
 }
 
 } // namespace ProcMacro
+
+ProcMacro::from_str_function_t __gccrs_pm_callback_from_str_fn = nullptr;

libgrust/libproc_macro/proc_macro.h

@@ -30,6 +30,7 @@
 #include "group.h"
 #include "punct.h"
 #include "ident.h"
+#include "registration.h"
 
 namespace ProcMacro {
 
@@ -61,6 +62,9 @@ struct Bang
   const char *name;
   BangMacro macro;
 };
+
+void
+proc_macro_register_from_str (ProcMacro::from_str_function_t fn);
 }
 
 enum ProcmacroTag

libgrust/libproc_macro/registration.h

@@ -0,0 +1,37 @@
+// Copyright (C) 2023 Free Software Foundation, Inc.
+//
+// This file is part of the GNU Proc Macro Library.  This library is free
+// software; you can redistribute it and/or modify it under the
+// terms of the GNU General Public License as published by the
+// Free Software Foundation; either version 3, or (at your option)
+// any later version.
+
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
+
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
+// <http://www.gnu.org/licenses/>.
+
+#ifndef REGISTRATION_H
+#define REGISTRATION_H
+
+#include <string>
+#include "tokenstream.h"
+
+namespace ProcMacro {
+
+using from_str_function_t = ProcMacro::TokenStream (*) (std::string &, bool &);
+
+} // namespace ProcMacro
+
+extern "C" ProcMacro::from_str_function_t __gccrs_pm_callback_from_str_fn;
+
+#endif /* !REGISTRATION_H */

libgrust/libproc_macro/tokenstream.cc

@@ -22,6 +22,7 @@
 
 #include "tokenstream.h"
 #include "tokentree.h"
+#include "registration.h"
 
 #include <cstring>
 
@@ -45,6 +46,12 @@ TokenStream::make_tokenstream (std::uint64_t capacity)
   return {data, 0, capacity};
 }
 
+TokenStream
+TokenStream::make_tokenstream (std::string &source, bool &has_error)
+{
+  return __gccrs_pm_callback_from_str_fn (source, has_error);
+}
+
 void
 TokenStream::grow (std::uint64_t delta)
 {
@@ -99,8 +106,11 @@ extern "C" bool
 TokenStream__from_string (unsigned char *str, std::uint64_t len,
 			  TokenStream *ts)
 {
-  // FIXME: Implement using parser ?
-  return false;
+  bool result;
+  auto source = std::string (reinterpret_cast<const char *> (str), len);
+
+  *ts = TokenStream::make_tokenstream (source, result);
+  return result;
 }
 
 extern "C" TokenStream

libgrust/libproc_macro/tokenstream.h

@@ -25,6 +25,7 @@
 
 #include <cstdint>
 #include <vector>
+#include <string>
 
 namespace ProcMacro {
 struct TokenTree;
@@ -43,6 +44,7 @@ public:
 
   static TokenStream make_tokenstream (std::vector<TokenTree> vec);
   static TokenStream make_tokenstream (std::uint64_t capacity = 1);
+  static TokenStream make_tokenstream (std::string &str, bool &has_error);
 
   static void drop (TokenStream *stream);
 };